Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29   
 30  __revision__ = "src/engine/SCons/Executor.py 5023 2010/06/14 22:05:46 scons" 
 31   
 32  import collections 
 33   
 34  from SCons.Debug import logInstanceCreation 
 35  import SCons.Errors 
 36  import SCons.Memoize 
 37   
 38   
 39 -class Batch(object): 
 40      """Remembers exact association between targets 
 41      and sources of executor.""" 
 42 -    def __init__(self, targets=[], sources=[]): 
 43          self.targets = targets 
 44          self.sources = sources 
 45   
 46   
 47   
 48 -class TSList(collections.UserList): 
 49      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 50      an executor Method.  This class is used in the Executor.lvars() 
 51      to delay creation of NodeList objects until they're needed. 
 52   
 53      Note that we subclass collections.UserList purely so that the 
 54      is_Sequence() function will identify an object of this class as 
 55      a list during variable expansion.  We're not really using any 
 56      collections.UserList methods in practice. 
 57      """ 
 58 -    def __init__(self, func): 
 59          self.func = func 
 60 -    def __getattr__(self, attr): 
 61          nl = self.func() 
 62          return getattr(nl, attr) 
 63 -    def __getitem__(self, i): 
 64          nl = self.func() 
 65          return nl[i] 
 66 -    def __getslice__(self, i, j): 
 67          nl = self.func() 
 68          i = max(i, 0); j = max(j, 0) 
 69          return nl[i:j] 
 70 -    def __str__(self): 
 71          nl = self.func() 
 72          return str(nl) 
 73 -    def __repr__(self): 
 74          nl = self.func() 
 75          return repr(nl) 
 76   
 77 -class TSObject(object): 
 78      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 79      an Executor method. 
 80      """ 
 81 -    def __init__(self, func): 
 82          self.func = func 
 83 -    def __getattr__(self, attr): 
 84          n = self.func() 
 85          return getattr(n, attr) 
 86 -    def __str__(self): 
 87          n = self.func() 
 88          if n: 
 89              return str(n) 
 90          return '' 
 91 -    def __repr__(self): 
 92          n = self.func() 
 93          if n: 
 94              return repr(n) 
 95          return '' 
 96   
 97 -def rfile(node): 
 98      """ 
 99      A function to return the results of a Node's rfile() method, 
100      if it exists, and the Node itself otherwise (if it's a Value 
101      Node, e.g.). 
102      """ 
103      try: 
104          rfile = node.rfile 
105      except AttributeError: 
106          return node 
107      else: 
108          return rfile() 
109   
110   
111 -class Executor(object): 
112      """A class for controlling instances of executing an action. 
113   
114      This largely exists to hold a single association of an action, 
115      environment, list of environment override dictionaries, targets 
116      and sources for later processing as needed. 
117      """ 
118   
119      if SCons.Memoize.use_memoizer: 
120          __metaclass__ = SCons.Memoize.Memoized_Metaclass 
121   
122      memoizer_counters = [] 
123   
124 -    def __init__(self, action, env=None, overridelist=[{}], 
125                   targets=[], sources=[], builder_kw={}): 
126          if __debug__: logInstanceCreation(self, 'Executor.Executor') 
127          self.set_action_list(action) 
128          self.pre_actions = [] 
129          self.post_actions = [] 
130          self.env = env 
131          self.overridelist = overridelist 
132          if targets or sources: 
133              self.batches = [Batch(targets[:], sources[:])] 
134          else: 
135              self.batches = [] 
136          self.builder_kw = builder_kw 
137          self._memo = {} 
138   
139 -    def get_lvars(self): 
140          try: 
141              return self.lvars 
142          except AttributeError: 
143              self.lvars = { 
144                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
145                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
146                  'SOURCE' : TSObject(self._get_source), 
147                  'SOURCES' : TSList(self._get_sources), 
148                  'TARGET' : TSObject(self._get_target), 
149                  'TARGETS' : TSList(self._get_targets), 
150                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
151                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
152              } 
153              return self.lvars 
154   
155 -    def _get_changes(self): 
156          cs = [] 
157          ct = [] 
158          us = [] 
159          ut = [] 
160          for b in self.batches: 
161              if b.targets[0].is_up_to_date(): 
162                  us.extend(list(map(rfile, b.sources))) 
163                  ut.extend(b.targets) 
164              else: 
165                  cs.extend(list(map(rfile, b.sources))) 
166                  ct.extend(b.targets) 
167          self._changed_sources_list = SCons.Util.NodeList(cs) 
168          self._changed_targets_list = SCons.Util.NodeList(ct) 
169          self._unchanged_sources_list = SCons.Util.NodeList(us) 
170          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
171   
172 -    def _get_changed_sources(self, *args, **kw): 
173          try: 
174              return self._changed_sources_list 
175          except AttributeError: 
176              self._get_changes() 
177              return self._changed_sources_list 
178   
179 -    def _get_changed_targets(self, *args, **kw): 
180          try: 
181              return self._changed_targets_list 
182          except AttributeError: 
183              self._get_changes() 
184              return self._changed_targets_list 
185   
186 -    def _get_source(self, *args, **kw): 
187          #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) 
188          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
189   
190 -    def _get_sources(self, *args, **kw): 
191          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
192   
193 -    def _get_target(self, *args, **kw): 
194          #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) 
195          return self.batches[0].targets[0].get_subst_proxy() 
196   
197 -    def _get_targets(self, *args, **kw): 
198          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
199   
200 -    def _get_unchanged_sources(self, *args, **kw): 
201          try: 
202              return self._unchanged_sources_list 
203          except AttributeError: 
204              self._get_changes() 
205              return self._unchanged_sources_list 
206   
207 -    def _get_unchanged_targets(self, *args, **kw): 
208          try: 
209              return self._unchanged_targets_list 
210          except AttributeError: 
211              self._get_changes() 
212              return self._unchanged_targets_list 
213   
214 -    def get_action_targets(self): 
215          if not self.action_list: 
216              return [] 
217          targets_string = self.action_list[0].get_targets(self.env, self) 
218          if targets_string[0] == '$': 
219              targets_string = targets_string[1:] 
220          return self.get_lvars()[targets_string] 
221   
222 -    def set_action_list(self, action): 
223          import SCons.Util 
224          if not SCons.Util.is_List(action): 
225              if not action: 
226                  import SCons.Errors 
227                  raise SCons.Errors.UserError("Executor must have an action.") 
228              action = [action] 
229          self.action_list = action 
230   
231 -    def get_action_list(self): 
232          return self.pre_actions + self.action_list + self.post_actions 
233   
234 -    def get_all_targets(self): 
235          """Returns all targets for all batches of this Executor.""" 
236          result = [] 
237          for batch in self.batches: 
238              result.extend(batch.targets) 
239          return result 
240   
241 -    def get_all_sources(self): 
242          """Returns all sources for all batches of this Executor.""" 
243          result = [] 
244          for batch in self.batches: 
245              result.extend(batch.sources) 
246          return result 
247   
248 -    def get_all_children(self): 
249          """Returns all unique children (dependencies) for all batches 
250          of this Executor. 
251   
252          The Taskmaster can recognize when it's already evaluated a 
253          Node, so we don't have to make this list unique for its intended 
254          canonical use case, but we expect there to be a lot of redundancy 
255          (long lists of batched .cc files #including the same .h files 
256          over and over), so removing the duplicates once up front should 
257          save the Taskmaster a lot of work. 
258          """ 
259          result = SCons.Util.UniqueList([]) 
260          for target in self.get_all_targets(): 
261              result.extend(target.children()) 
262          return result 
263   
264 -    def get_all_prerequisites(self): 
265          """Returns all unique (order-only) prerequisites for all batches 
266          of this Executor. 
267          """ 
268          result = SCons.Util.UniqueList([]) 
269          for target in self.get_all_targets(): 
270              result.extend(target.prerequisites) 
271          return result 
272   
273 -    def get_action_side_effects(self): 
274   
275          """Returns all side effects for all batches of this 
276          Executor used by the underlying Action. 
277          """ 
278          result = SCons.Util.UniqueList([]) 
279          for target in self.get_action_targets(): 
280              result.extend(target.side_effects) 
281          return result 
282   
283      memoizer_counters.append(SCons.Memoize.CountValue('get_build_env')) 
284   
285 -    def get_build_env(self): 
286          """Fetch or create the appropriate build Environment 
287          for this Executor. 
288          """ 
289          try: 
290              return self._memo['get_build_env'] 
291          except KeyError: 
292              pass 
293   
294          # Create the build environment instance with appropriate 
295          # overrides.  These get evaluated against the current 
296          # environment's construction variables so that users can 
297          # add to existing values by referencing the variable in 
298          # the expansion. 
299          overrides = {} 
300          for odict in self.overridelist: 
301              overrides.update(odict) 
302   
303          import SCons.Defaults 
304          env = self.env or SCons.Defaults.DefaultEnvironment() 
305          build_env = env.Override(overrides) 
306   
307          self._memo['get_build_env'] = build_env 
308   
309          return build_env 
310   
311 -    def get_build_scanner_path(self, scanner): 
312          """Fetch the scanner path for this executor's targets and sources. 
313          """ 
314          env = self.get_build_env() 
315          try: 
316              cwd = self.batches[0].targets[0].cwd 
317          except (IndexError, AttributeError): 
318              cwd = None 
319          return scanner.path(env, cwd, 
320                              self.get_all_targets(), 
321                              self.get_all_sources()) 
322   
323 -    def get_kw(self, kw={}): 
324          result = self.builder_kw.copy() 
325          result.update(kw) 
326          result['executor'] = self 
327          return result 
328   
329 -    def do_nothing(self, target, kw): 
330          return 0 
331   
332 -    def do_execute(self, target, kw): 
333          """Actually execute the action list.""" 
334          env = self.get_build_env() 
335          kw = self.get_kw(kw) 
336          status = 0 
337          for act in self.get_action_list(): 
338              #args = (self.get_all_targets(), self.get_all_sources(), env) 
339              args = ([], [], env) 
340              status = act(*args, **kw) 
341              if isinstance(status, SCons.Errors.BuildError): 
342                  status.executor = self 
343                  raise status 
344              elif status: 
345                  msg = "Error %s" % status 
346                  raise SCons.Errors.BuildError( 
347                      errstr=msg,  
348                      node=self.batches[0].targets, 
349                      executor=self,  
350                      action=act) 
351          return status 
352   
353      # use extra indirection because with new-style objects (Python 2.2 
354      # and above) we can't override special methods, and nullify() needs 
355      # to be able to do this. 
356   
357 -    def __call__(self, target, **kw): 
358          return self.do_execute(target, kw) 
359   
360 -    def cleanup(self): 
361          self._memo = {} 
362   
363 -    def add_sources(self, sources): 
364          """Add source files to this Executor's list.  This is necessary 
365          for "multi" Builders that can be called repeatedly to build up 
366          a source file list for a given target.""" 
367          # TODO(batch):  extend to multiple batches 
368          assert (len(self.batches) == 1) 
369          # TODO(batch):  remove duplicates? 
370          sources = [x for x in sources if x not in self.batches[0].sources] 
371          self.batches[0].sources.extend(sources) 
372   
373 -    def get_sources(self): 
374          return self.batches[0].sources 
375   
376 -    def add_batch(self, targets, sources): 
377          """Add pair of associated target and source to this Executor's list. 
378          This is necessary for "batch" Builders that can be called repeatedly 
379          to build up a list of matching target and source files that will be 
380          used in order to update multiple target files at once from multiple 
381          corresponding source files, for tools like MSVC that support it.""" 
382          self.batches.append(Batch(targets, sources)) 
383   
384 -    def prepare(self): 
385          """ 
386          Preparatory checks for whether this Executor can go ahead 
387          and (try to) build its targets. 
388          """ 
389          for s in self.get_all_sources(): 
390              if s.missing(): 
391                  msg = "Source `%s' not found, needed by target `%s'." 
392                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
393   
394 -    def add_pre_action(self, action): 
395          self.pre_actions.append(action) 
396   
397 -    def add_post_action(self, action): 
398          self.post_actions.append(action) 
399   
400      # another extra indirection for new-style objects and nullify... 
401   
402 -    def my_str(self): 
403          env = self.get_build_env() 
404          return "\n".join([action.genstring(self.get_all_targets(), 
405                                             self.get_all_sources(), 
406                                             env) 
407                            for action in self.get_action_list()]) 
408   
409   
410 -    def __str__(self): 
411          return self.my_str() 
412   
413 -    def nullify(self): 
414          self.cleanup() 
415          self.do_execute = self.do_nothing 
416          self.my_str     = lambda: '' 
417   
418      memoizer_counters.append(SCons.Memoize.CountValue('get_contents')) 
419   
420 -    def get_contents(self): 
421          """Fetch the signature contents.  This is the main reason this 
422          class exists, so we can compute this once and cache it regardless 
423          of how many target or source Nodes there are. 
424          """ 
425          try: 
426              return self._memo['get_contents'] 
427          except KeyError: 
428              pass 
429          env = self.get_build_env() 
430          result = "".join([action.get_contents(self.get_all_targets(), 
431                                                self.get_all_sources(), 
432                                                env) 
433                            for action in self.get_action_list()]) 
434          self._memo['get_contents'] = result 
435          return result 
436   
437 -    def get_timestamp(self): 
438          """Fetch a time stamp for this Executor.  We don't have one, of 
439          course (only files do), but this is the interface used by the 
440          timestamp module. 
441          """ 
442          return 0 
443   
444 -    def scan_targets(self, scanner): 
445          # TODO(batch):  scan by batches 
446          self.scan(scanner, self.get_all_targets()) 
447   
448 -    def scan_sources(self, scanner): 
449          # TODO(batch):  scan by batches 
450          if self.batches[0].sources: 
451              self.scan(scanner, self.get_all_sources()) 
452   
453 -    def scan(self, scanner, node_list): 
454          """Scan a list of this Executor's files (targets or sources) for 
455          implicit dependencies and update all of the targets with them. 
456          This essentially short-circuits an N*M scan of the sources for 
457          each individual target, which is a hell of a lot more efficient. 
458          """ 
459          env = self.get_build_env() 
460   
461          # TODO(batch):  scan by batches) 
462          deps = [] 
463          if scanner: 
464              for node in node_list: 
465                  node.disambiguate() 
466                  s = scanner.select(node) 
467                  if not s: 
468                      continue 
469                  path = self.get_build_scanner_path(s) 
470                  deps.extend(node.get_implicit_deps(env, s, path)) 
471          else: 
472              kw = self.get_kw() 
473              for node in node_list: 
474                  node.disambiguate() 
475                  scanner = node.get_env_scanner(env, kw) 
476                  if not scanner: 
477                      continue 
478                  scanner = scanner.select(node) 
479                  if not scanner: 
480                      continue 
481                  path = self.get_build_scanner_path(scanner) 
482                  deps.extend(node.get_implicit_deps(env, scanner, path)) 
483   
484          deps.extend(self.get_implicit_deps()) 
485   
486          for tgt in self.get_all_targets(): 
487              tgt.add_to_implicit(deps) 
488   
489 -    def _get_unignored_sources_key(self, node, ignore=()): 
490          return (node,) + tuple(ignore) 
491   
492      memoizer_counters.append(SCons.Memoize.CountDict('get_unignored_sources', _get_unignored_sources_key)) 
493   
494 -    def get_unignored_sources(self, node, ignore=()): 
495          key = (node,) + tuple(ignore) 
496          try: 
497              memo_dict = self._memo['get_unignored_sources'] 
498          except KeyError: 
499              memo_dict = {} 
500              self._memo['get_unignored_sources'] = memo_dict 
501          else: 
502              try: 
503                  return memo_dict[key] 
504              except KeyError: 
505                  pass 
506   
507          if node: 
508              # TODO:  better way to do this (it's a linear search, 
509              # but it may not be critical path)? 
510              sourcelist = [] 
511              for b in self.batches: 
512                  if node in b.targets: 
513                      sourcelist = b.sources 
514                      break 
515          else: 
516              sourcelist = self.get_all_sources() 
517          if ignore: 
518              idict = {} 
519              for i in ignore: 
520                  idict[i] = 1 
521              sourcelist = [s for s in sourcelist if s not in idict] 
522   
523          memo_dict[key] = sourcelist 
524   
525          return sourcelist 
526   
527 -    def get_implicit_deps(self): 
528          """Return the executor's implicit dependencies, i.e. the nodes of 
529          the commands to be executed.""" 
530          result = [] 
531          build_env = self.get_build_env() 
532          for act in self.get_action_list(): 
533              deps = act.get_implicit_deps(self.get_all_targets(), 
534                                           self.get_all_sources(), 
535                                           build_env) 
536              result.extend(deps) 
537          return result 
538   
539   
540   
541  _batch_executors = {} 
542   
543 -def GetBatchExecutor(key): 
544      return _batch_executors[key] 
545   
546 -def AddBatchExecutor(key, executor): 
547      assert key not in _batch_executors 
548      _batch_executors[key] = executor 
549   
550  nullenv = None 
551   
552   
553 -def get_NullEnvironment(): 
554      """Use singleton pattern for Null Environments.""" 
555      global nullenv 
556   
557      import SCons.Util 
558      class NullEnvironment(SCons.Util.Null): 
559          import SCons.CacheDir 
560          _CacheDir_path = None 
561          _CacheDir = SCons.CacheDir.CacheDir(None) 
562          def get_CacheDir(self): 
563              return self._CacheDir 
564   
565      if not nullenv: 
566          nullenv = NullEnvironment() 
567      return nullenv 
568   
569 -class Null(object): 
570      """A null Executor, with a null build Environment, that does 
571      nothing when the rest of the methods call it. 
572   
573      This might be able to disapper when we refactor things to 
574      disassociate Builders from Nodes entirely, so we're not 
575      going to worry about unit tests for this--at least for now. 
576      """ 
577 -    def __init__(self, *args, **kw): 
578          if __debug__: logInstanceCreation(self, 'Executor.Null') 
579          self.batches = [Batch(kw['targets'][:], [])] 
580 -    def get_build_env(self): 
581          return get_NullEnvironment() 
582 -    def get_build_scanner_path(self): 
583          return None 
584 -    def cleanup(self): 
585          pass 
586 -    def prepare(self): 
587          pass 
588 -    def get_unignored_sources(self, *args, **kw): 
589          return tuple(()) 
590 -    def get_action_targets(self): 
591          return [] 
592 -    def get_action_list(self): 
593          return [] 
594 -    def get_all_targets(self): 
595          return self.batches[0].targets 
596 -    def get_all_sources(self): 
597          return self.batches[0].targets[0].sources 
598 -    def get_all_children(self): 
599          return self.get_all_sources() 
600 -    def get_all_prerequisites(self): 
601          return [] 
602 -    def get_action_side_effects(self): 
603          return [] 
604 -    def __call__(self, *args, **kw): 
605          return 0 
606 -    def get_contents(self): 
607          return '' 
608 -    def _morph(self): 
609          """Morph this Null executor to a real Executor object.""" 
610          batches = self.batches 
611          self.__class__ = Executor 
612          self.__init__([])             
613          self.batches = batches 
614   
615      # The following methods require morphing this Null Executor to a 
616      # real Executor object. 
617   
618 -    def add_pre_action(self, action): 
619          self._morph() 
620          self.add_pre_action(action) 
621 -    def add_post_action(self, action): 
622          self._morph() 
623          self.add_post_action(action) 
624 -    def set_action_list(self, action): 
625          self._morph() 
626          self.set_action_list(action) 
627   
628   
629  # Local Variables: 
630  # tab-width:4 
631  # indent-tabs-mode:nil 
632  # End: 
633  # vim: set expandtab tabstop=4 shiftwidth=4: 
634