Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2015 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29   
 30  __revision__ = "src/engine/SCons/Executor.py rel_2.4.0:3365:9259ea1c13d7 2015/09/21 14:03:43 bdbaddog" 
 31   
 32  import collections 
 33   
 34  import SCons.Debug 
 35  from SCons.Debug import logInstanceCreation 
 36  import SCons.Errors 
 37  import SCons.Memoize 
 38   
 39   
 40 -class Batch(object): 
 41      """Remembers exact association between targets 
 42      and sources of executor.""" 
 43       
 44      __slots__ = ('targets', 
 45                   'sources') 
 46       
 47 -    def __init__(self, targets=[], sources=[]): 
 48          self.targets = targets 
 49          self.sources = sources 
 50   
 51   
 52   
 53 -class TSList(collections.UserList): 
 54      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 55      an executor Method.  This class is used in the Executor.lvars() 
 56      to delay creation of NodeList objects until they're needed. 
 57   
 58      Note that we subclass collections.UserList purely so that the 
 59      is_Sequence() function will identify an object of this class as 
 60      a list during variable expansion.  We're not really using any 
 61      collections.UserList methods in practice. 
 62      """ 
 63 -    def __init__(self, func): 
 64          self.func = func 
 65 -    def __getattr__(self, attr): 
 66          nl = self.func() 
 67          return getattr(nl, attr) 
 68 -    def __getitem__(self, i): 
 69          nl = self.func() 
 70          return nl[i] 
 71 -    def __getslice__(self, i, j): 
 72          nl = self.func() 
 73          i = max(i, 0); j = max(j, 0) 
 74          return nl[i:j] 
 75 -    def __str__(self): 
 76          nl = self.func() 
 77          return str(nl) 
 78 -    def __repr__(self): 
 79          nl = self.func() 
 80          return repr(nl) 
 81   
 82 -class TSObject(object): 
 83      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 84      an Executor method. 
 85      """ 
 86 -    def __init__(self, func): 
 87          self.func = func 
 88 -    def __getattr__(self, attr): 
 89          n = self.func() 
 90          return getattr(n, attr) 
 91 -    def __str__(self): 
 92          n = self.func() 
 93          if n: 
 94              return str(n) 
 95          return '' 
 96 -    def __repr__(self): 
 97          n = self.func() 
 98          if n: 
 99              return repr(n) 
100          return '' 
101   
102 -def rfile(node): 
103      """ 
104      A function to return the results of a Node's rfile() method, 
105      if it exists, and the Node itself otherwise (if it's a Value 
106      Node, e.g.). 
107      """ 
108      try: 
109          rfile = node.rfile 
110      except AttributeError: 
111          return node 
112      else: 
113          return rfile() 
114   
115   
116 -def execute_nothing(obj, target, kw): 
117      return 0 
118   
119 -def execute_action_list(obj, target, kw): 
120      """Actually execute the action list.""" 
121      env = obj.get_build_env() 
122      kw = obj.get_kw(kw) 
123      status = 0 
124      for act in obj.get_action_list(): 
125          #args = (self.get_all_targets(), self.get_all_sources(), env) 
126          args = ([], [], env) 
127          status = act(*args, **kw) 
128          if isinstance(status, SCons.Errors.BuildError): 
129              status.executor = obj 
130              raise status 
131          elif status: 
132              msg = "Error %s" % status 
133              raise SCons.Errors.BuildError( 
134                  errstr=msg,  
135                  node=obj.batches[0].targets, 
136                  executor=obj,  
137                  action=act) 
138      return status 
139   
140  _do_execute_map = {0 : execute_nothing, 
141                     1 : execute_action_list} 
142   
143   
144 -def execute_actions_str(obj): 
145      env = obj.get_build_env() 
146      return "\n".join([action.genstring(obj.get_all_targets(), 
147                                         obj.get_all_sources(), 
148                                         env) 
149                        for action in obj.get_action_list()]) 
150   
151 -def execute_null_str(obj): 
152      return '' 
153   
154  _execute_str_map = {0 : execute_null_str, 
155                      1 : execute_actions_str} 
156   
157   
158 -class Executor(object): 
159      """A class for controlling instances of executing an action. 
160   
161      This largely exists to hold a single association of an action, 
162      environment, list of environment override dictionaries, targets 
163      and sources for later processing as needed. 
164      """ 
165   
166      __slots__ = ('pre_actions', 
167                   'post_actions', 
168                   'env', 
169                   'overridelist', 
170                   'batches', 
171                   'builder_kw', 
172                   '_memo', 
173                   'lvars', 
174                   '_changed_sources_list', 
175                   '_changed_targets_list', 
176                   '_unchanged_sources_list', 
177                   '_unchanged_targets_list', 
178                   'action_list', 
179                   '_do_execute', 
180                   '_execute_str') 
181   
182 -    def __init__(self, action, env=None, overridelist=[{}], 
183                   targets=[], sources=[], builder_kw={}): 
184          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 
185          self.set_action_list(action) 
186          self.pre_actions = [] 
187          self.post_actions = [] 
188          self.env = env 
189          self.overridelist = overridelist 
190          if targets or sources: 
191              self.batches = [Batch(targets[:], sources[:])] 
192          else: 
193              self.batches = [] 
194          self.builder_kw = builder_kw 
195          self._do_execute = 1 
196          self._execute_str = 1 
197          self._memo = {} 
198   
199 -    def get_lvars(self): 
200          try: 
201              return self.lvars 
202          except AttributeError: 
203              self.lvars = { 
204                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
205                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
206                  'SOURCE' : TSObject(self._get_source), 
207                  'SOURCES' : TSList(self._get_sources), 
208                  'TARGET' : TSObject(self._get_target), 
209                  'TARGETS' : TSList(self._get_targets), 
210                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
211                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
212              } 
213              return self.lvars 
214   
215 -    def _get_changes(self): 
216          cs = [] 
217          ct = [] 
218          us = [] 
219          ut = [] 
220          for b in self.batches: 
221              if b.targets[0].is_up_to_date(): 
222                  us.extend(list(map(rfile, b.sources))) 
223                  ut.extend(b.targets) 
224              else: 
225                  cs.extend(list(map(rfile, b.sources))) 
226                  ct.extend(b.targets) 
227          self._changed_sources_list = SCons.Util.NodeList(cs) 
228          self._changed_targets_list = SCons.Util.NodeList(ct) 
229          self._unchanged_sources_list = SCons.Util.NodeList(us) 
230          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
231   
232 -    def _get_changed_sources(self, *args, **kw): 
233          try: 
234              return self._changed_sources_list 
235          except AttributeError: 
236              self._get_changes() 
237              return self._changed_sources_list 
238   
239 -    def _get_changed_targets(self, *args, **kw): 
240          try: 
241              return self._changed_targets_list 
242          except AttributeError: 
243              self._get_changes() 
244              return self._changed_targets_list 
245   
246 -    def _get_source(self, *args, **kw): 
247          #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) 
248          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
249   
250 -    def _get_sources(self, *args, **kw): 
251          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
252   
253 -    def _get_target(self, *args, **kw): 
254          #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) 
255          return self.batches[0].targets[0].get_subst_proxy() 
256   
257 -    def _get_targets(self, *args, **kw): 
258          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
259   
260 -    def _get_unchanged_sources(self, *args, **kw): 
261          try: 
262              return self._unchanged_sources_list 
263          except AttributeError: 
264              self._get_changes() 
265              return self._unchanged_sources_list 
266   
267 -    def _get_unchanged_targets(self, *args, **kw): 
268          try: 
269              return self._unchanged_targets_list 
270          except AttributeError: 
271              self._get_changes() 
272              return self._unchanged_targets_list 
273   
274 -    def get_action_targets(self): 
275          if not self.action_list: 
276              return [] 
277          targets_string = self.action_list[0].get_targets(self.env, self) 
278          if targets_string[0] == '$': 
279              targets_string = targets_string[1:] 
280          return self.get_lvars()[targets_string] 
281   
282 -    def set_action_list(self, action): 
283          import SCons.Util 
284          if not SCons.Util.is_List(action): 
285              if not action: 
286                  import SCons.Errors 
287                  raise SCons.Errors.UserError("Executor must have an action.") 
288              action = [action] 
289          self.action_list = action 
290   
291 -    def get_action_list(self): 
292          if self.action_list is None: 
293              return [] 
294          return self.pre_actions + self.action_list + self.post_actions 
295   
296 -    def get_all_targets(self): 
297          """Returns all targets for all batches of this Executor.""" 
298          result = [] 
299          for batch in self.batches: 
300              result.extend(batch.targets) 
301          return result 
302   
303 -    def get_all_sources(self): 
304          """Returns all sources for all batches of this Executor.""" 
305          result = [] 
306          for batch in self.batches: 
307              result.extend(batch.sources) 
308          return result 
309   
310 -    def get_all_children(self): 
311          """Returns all unique children (dependencies) for all batches 
312          of this Executor. 
313   
314          The Taskmaster can recognize when it's already evaluated a 
315          Node, so we don't have to make this list unique for its intended 
316          canonical use case, but we expect there to be a lot of redundancy 
317          (long lists of batched .cc files #including the same .h files 
318          over and over), so removing the duplicates once up front should 
319          save the Taskmaster a lot of work. 
320          """ 
321          result = SCons.Util.UniqueList([]) 
322          for target in self.get_all_targets(): 
323              result.extend(target.children()) 
324          return result 
325   
326 -    def get_all_prerequisites(self): 
327          """Returns all unique (order-only) prerequisites for all batches 
328          of this Executor. 
329          """ 
330          result = SCons.Util.UniqueList([]) 
331          for target in self.get_all_targets(): 
332              if target.prerequisites is not None: 
333                  result.extend(target.prerequisites) 
334          return result 
335   
336 -    def get_action_side_effects(self): 
337   
338          """Returns all side effects for all batches of this 
339          Executor used by the underlying Action. 
340          """ 
341          result = SCons.Util.UniqueList([]) 
342          for target in self.get_action_targets(): 
343              result.extend(target.side_effects) 
344          return result 
345   
346      @SCons.Memoize.CountMethodCall 
347 -    def get_build_env(self): 
348          """Fetch or create the appropriate build Environment 
349          for this Executor. 
350          """ 
351          try: 
352              return self._memo['get_build_env'] 
353          except KeyError: 
354              pass 
355   
356          # Create the build environment instance with appropriate 
357          # overrides.  These get evaluated against the current 
358          # environment's construction variables so that users can 
359          # add to existing values by referencing the variable in 
360          # the expansion. 
361          overrides = {} 
362          for odict in self.overridelist: 
363              overrides.update(odict) 
364   
365          import SCons.Defaults 
366          env = self.env or SCons.Defaults.DefaultEnvironment() 
367          build_env = env.Override(overrides) 
368   
369          self._memo['get_build_env'] = build_env 
370   
371          return build_env 
372   
373 -    def get_build_scanner_path(self, scanner): 
374          """Fetch the scanner path for this executor's targets and sources. 
375          """ 
376          env = self.get_build_env() 
377          try: 
378              cwd = self.batches[0].targets[0].cwd 
379          except (IndexError, AttributeError): 
380              cwd = None 
381          return scanner.path(env, cwd, 
382                              self.get_all_targets(), 
383                              self.get_all_sources()) 
384   
385 -    def get_kw(self, kw={}): 
386          result = self.builder_kw.copy() 
387          result.update(kw) 
388          result['executor'] = self 
389          return result 
390   
391      # use extra indirection because with new-style objects (Python 2.2 
392      # and above) we can't override special methods, and nullify() needs 
393      # to be able to do this. 
394   
395 -    def __call__(self, target, **kw): 
396          return _do_execute_map[self._do_execute](self, target, kw) 
397   
398 -    def cleanup(self): 
399          self._memo = {} 
400   
401 -    def add_sources(self, sources): 
402          """Add source files to this Executor's list.  This is necessary 
403          for "multi" Builders that can be called repeatedly to build up 
404          a source file list for a given target.""" 
405          # TODO(batch):  extend to multiple batches 
406          assert (len(self.batches) == 1) 
407          # TODO(batch):  remove duplicates? 
408          sources = [x for x in sources if x not in self.batches[0].sources] 
409          self.batches[0].sources.extend(sources) 
410   
411 -    def get_sources(self): 
412          return self.batches[0].sources 
413   
414 -    def add_batch(self, targets, sources): 
415          """Add pair of associated target and source to this Executor's list. 
416          This is necessary for "batch" Builders that can be called repeatedly 
417          to build up a list of matching target and source files that will be 
418          used in order to update multiple target files at once from multiple 
419          corresponding source files, for tools like MSVC that support it.""" 
420          self.batches.append(Batch(targets, sources)) 
421   
422 -    def prepare(self): 
423          """ 
424          Preparatory checks for whether this Executor can go ahead 
425          and (try to) build its targets. 
426          """ 
427          for s in self.get_all_sources(): 
428              if s.missing(): 
429                  msg = "Source `%s' not found, needed by target `%s'." 
430                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
431   
432 -    def add_pre_action(self, action): 
433          self.pre_actions.append(action) 
434   
435 -    def add_post_action(self, action): 
436          self.post_actions.append(action) 
437   
438      # another extra indirection for new-style objects and nullify... 
439   
440 -    def __str__(self): 
441          return _execute_str_map[self._execute_str](self) 
442   
443 -    def nullify(self): 
444          self.cleanup() 
445          self._do_execute = 0 
446          self._execute_str = 0 
447   
448      @SCons.Memoize.CountMethodCall 
449 -    def get_contents(self): 
450          """Fetch the signature contents.  This is the main reason this 
451          class exists, so we can compute this once and cache it regardless 
452          of how many target or source Nodes there are. 
453          """ 
454          try: 
455              return self._memo['get_contents'] 
456          except KeyError: 
457              pass 
458          env = self.get_build_env() 
459          result = "".join([action.get_contents(self.get_all_targets(), 
460                                                self.get_all_sources(), 
461                                                env) 
462                            for action in self.get_action_list()]) 
463          self._memo['get_contents'] = result 
464          return result 
465   
466 -    def get_timestamp(self): 
467          """Fetch a time stamp for this Executor.  We don't have one, of 
468          course (only files do), but this is the interface used by the 
469          timestamp module. 
470          """ 
471          return 0 
472   
473 -    def scan_targets(self, scanner): 
474          # TODO(batch):  scan by batches 
475          self.scan(scanner, self.get_all_targets()) 
476   
477 -    def scan_sources(self, scanner): 
478          # TODO(batch):  scan by batches 
479          if self.batches[0].sources: 
480              self.scan(scanner, self.get_all_sources()) 
481   
482 -    def scan(self, scanner, node_list): 
483          """Scan a list of this Executor's files (targets or sources) for 
484          implicit dependencies and update all of the targets with them. 
485          This essentially short-circuits an N*M scan of the sources for 
486          each individual target, which is a hell of a lot more efficient. 
487          """ 
488          env = self.get_build_env() 
489   
490          # TODO(batch):  scan by batches) 
491          deps = [] 
492          if scanner: 
493              for node in node_list: 
494                  node.disambiguate() 
495                  s = scanner.select(node) 
496                  if not s: 
497                      continue 
498                  path = self.get_build_scanner_path(s) 
499                  deps.extend(node.get_implicit_deps(env, s, path)) 
500          else: 
501              kw = self.get_kw() 
502              for node in node_list: 
503                  node.disambiguate() 
504                  scanner = node.get_env_scanner(env, kw) 
505                  if not scanner: 
506                      continue 
507                  scanner = scanner.select(node) 
508                  if not scanner: 
509                      continue 
510                  path = self.get_build_scanner_path(scanner) 
511                  deps.extend(node.get_implicit_deps(env, scanner, path)) 
512   
513          deps.extend(self.get_implicit_deps()) 
514   
515          for tgt in self.get_all_targets(): 
516              tgt.add_to_implicit(deps) 
517   
518 -    def _get_unignored_sources_key(self, node, ignore=()): 
519          return (node,) + tuple(ignore) 
520   
521      @SCons.Memoize.CountDictCall(_get_unignored_sources_key) 
522 -    def get_unignored_sources(self, node, ignore=()): 
523          key = (node,) + tuple(ignore) 
524          try: 
525              memo_dict = self._memo['get_unignored_sources'] 
526          except KeyError: 
527              memo_dict = {} 
528              self._memo['get_unignored_sources'] = memo_dict 
529          else: 
530              try: 
531                  return memo_dict[key] 
532              except KeyError: 
533                  pass 
534   
535          if node: 
536              # TODO:  better way to do this (it's a linear search, 
537              # but it may not be critical path)? 
538              sourcelist = [] 
539              for b in self.batches: 
540                  if node in b.targets: 
541                      sourcelist = b.sources 
542                      break 
543          else: 
544              sourcelist = self.get_all_sources() 
545          if ignore: 
546              idict = {} 
547              for i in ignore: 
548                  idict[i] = 1 
549              sourcelist = [s for s in sourcelist if s not in idict] 
550   
551          memo_dict[key] = sourcelist 
552   
553          return sourcelist 
554   
555 -    def get_implicit_deps(self): 
556          """Return the executor's implicit dependencies, i.e. the nodes of 
557          the commands to be executed.""" 
558          result = [] 
559          build_env = self.get_build_env() 
560          for act in self.get_action_list(): 
561              deps = act.get_implicit_deps(self.get_all_targets(), 
562                                           self.get_all_sources(), 
563                                           build_env) 
564              result.extend(deps) 
565          return result 
566   
567   
568   
569  _batch_executors = {} 
570   
571 -def GetBatchExecutor(key): 
572      return _batch_executors[key] 
573   
574 -def AddBatchExecutor(key, executor): 
575      assert key not in _batch_executors 
576      _batch_executors[key] = executor 
577   
578  nullenv = None 
579   
580   
581  import SCons.Util 
582 -class NullEnvironment(SCons.Util.Null): 
583      import SCons.CacheDir 
584      _CacheDir_path = None 
585      _CacheDir = SCons.CacheDir.CacheDir(None) 
586 -    def get_CacheDir(self): 
587          return self._CacheDir 
588   
589   
590 -def get_NullEnvironment(): 
591      """Use singleton pattern for Null Environments.""" 
592      global nullenv 
593   
594      if nullenv is None: 
595          nullenv = NullEnvironment() 
596      return nullenv 
597   
598 -class Null(object): 
599      """A null Executor, with a null build Environment, that does 
600      nothing when the rest of the methods call it. 
601   
602      This might be able to disappear when we refactor things to 
603      disassociate Builders from Nodes entirely, so we're not 
604      going to worry about unit tests for this--at least for now. 
605      """ 
606       
607      __slots__ = ('pre_actions', 
608                   'post_actions', 
609                   'env', 
610                   'overridelist', 
611                   'batches', 
612                   'builder_kw', 
613                   '_memo', 
614                   'lvars', 
615                   '_changed_sources_list', 
616                   '_changed_targets_list', 
617                   '_unchanged_sources_list', 
618                   '_unchanged_targets_list', 
619                   'action_list', 
620                   '_do_execute', 
621                   '_execute_str') 
622       
623 -    def __init__(self, *args, **kw): 
624          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 
625          self.batches = [Batch(kw['targets'][:], [])] 
626 -    def get_build_env(self): 
627          return get_NullEnvironment() 
628 -    def get_build_scanner_path(self): 
629          return None 
630 -    def cleanup(self): 
631          pass 
632 -    def prepare(self): 
633          pass 
634 -    def get_unignored_sources(self, *args, **kw): 
635          return tuple(()) 
636 -    def get_action_targets(self): 
637          return [] 
638 -    def get_action_list(self): 
639          return [] 
640 -    def get_all_targets(self): 
641          return self.batches[0].targets 
642 -    def get_all_sources(self): 
643          return self.batches[0].targets[0].sources 
644 -    def get_all_children(self): 
645          return self.batches[0].targets[0].children() 
646 -    def get_all_prerequisites(self): 
647          return [] 
648 -    def get_action_side_effects(self): 
649          return [] 
650 -    def __call__(self, *args, **kw): 
651          return 0 
652 -    def get_contents(self): 
653          return '' 
654 -    def _morph(self): 
655          """Morph this Null executor to a real Executor object.""" 
656          batches = self.batches 
657          self.__class__ = Executor 
658          self.__init__([])             
659          self.batches = batches 
660   
661      # The following methods require morphing this Null Executor to a 
662      # real Executor object. 
663   
664 -    def add_pre_action(self, action): 
665          self._morph() 
666          self.add_pre_action(action) 
667 -    def add_post_action(self, action): 
668          self._morph() 
669          self.add_post_action(action) 
670 -    def set_action_list(self, action): 
671          self._morph() 
672          self.set_action_list(action) 
673   
674  # Local Variables: 
675  # tab-width:4 
676  # indent-tabs-mode:nil 
677  # End: 
678  # vim: set expandtab tabstop=4 shiftwidth=4: 
679