Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2017 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29  from __future__ import print_function 
 30   
 31  __revision__ = "src/engine/SCons/Executor.py 74b2c53bc42290e911b334a6b44f187da698a668 2017/11/14 13:16:53 bdbaddog" 
 32   
 33  import collections 
 34   
 35  import SCons.Debug 
 36  from SCons.Debug import logInstanceCreation 
 37  import SCons.Errors 
 38  import SCons.Memoize 
 39  from SCons.compat import with_metaclass, NoSlotsPyPy 
 40   
 41 -class Batch(object): 
 42      """Remembers exact association between targets 
 43      and sources of executor.""" 
 44       
 45      __slots__ = ('targets', 
 46                   'sources') 
 47       
 48 -    def __init__(self, targets=[], sources=[]): 
 49          self.targets = targets 
 50          self.sources = sources 
 51   
 52   
 53   
 54 -class TSList(collections.UserList): 
 55      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 56      an executor Method.  This class is used in the Executor.lvars() 
 57      to delay creation of NodeList objects until they're needed. 
 58   
 59      Note that we subclass collections.UserList purely so that the 
 60      is_Sequence() function will identify an object of this class as 
 61      a list during variable expansion.  We're not really using any 
 62      collections.UserList methods in practice. 
 63      """ 
 64 -    def __init__(self, func): 
 65          self.func = func 
 66 -    def __getattr__(self, attr): 
 67          nl = self.func() 
 68          return getattr(nl, attr) 
 69 -    def __getitem__(self, i): 
 70          nl = self.func() 
 71          return nl[i] 
 72 -    def __getslice__(self, i, j): 
 73          nl = self.func() 
 74          i = max(i, 0); j = max(j, 0) 
 75          return nl[i:j] 
 76 -    def __str__(self): 
 77          nl = self.func() 
 78          return str(nl) 
 79 -    def __repr__(self): 
 80          nl = self.func() 
 81          return repr(nl) 
 82   
 83 -class TSObject(object): 
 84      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 85      an Executor method. 
 86      """ 
 87 -    def __init__(self, func): 
 88          self.func = func 
 89 -    def __getattr__(self, attr): 
 90          n = self.func() 
 91          return getattr(n, attr) 
 92 -    def __str__(self): 
 93          n = self.func() 
 94          if n: 
 95              return str(n) 
 96          return '' 
 97 -    def __repr__(self): 
 98          n = self.func() 
 99          if n: 
100              return repr(n) 
101          return '' 
102   
103 -def rfile(node): 
104      """ 
105      A function to return the results of a Node's rfile() method, 
106      if it exists, and the Node itself otherwise (if it's a Value 
107      Node, e.g.). 
108      """ 
109      try: 
110          rfile = node.rfile 
111      except AttributeError: 
112          return node 
113      else: 
114          return rfile() 
115   
116   
117 -def execute_nothing(obj, target, kw): 
118      return 0 
119   
120 -def execute_action_list(obj, target, kw): 
121      """Actually execute the action list.""" 
122      env = obj.get_build_env() 
123      kw = obj.get_kw(kw) 
124      status = 0 
125      for act in obj.get_action_list(): 
126          args = ([], [], env) 
127          status = act(*args, **kw) 
128          if isinstance(status, SCons.Errors.BuildError): 
129              status.executor = obj 
130              raise status 
131          elif status: 
132              msg = "Error %s" % status 
133              raise SCons.Errors.BuildError( 
134                  errstr=msg,  
135                  node=obj.batches[0].targets, 
136                  executor=obj,  
137                  action=act) 
138      return status 
139   
140  _do_execute_map = {0 : execute_nothing, 
141                     1 : execute_action_list} 
142   
143   
144 -def execute_actions_str(obj): 
145      env = obj.get_build_env() 
146      return "\n".join([action.genstring(obj.get_all_targets(), 
147                                         obj.get_all_sources(), 
148                                         env) 
149                        for action in obj.get_action_list()]) 
150   
151 -def execute_null_str(obj): 
152      return '' 
153   
154  _execute_str_map = {0 : execute_null_str, 
155                      1 : execute_actions_str} 
156   
157   
158 -class Executor(object, with_metaclass(NoSlotsPyPy)): 
159      """A class for controlling instances of executing an action. 
160   
161      This largely exists to hold a single association of an action, 
162      environment, list of environment override dictionaries, targets 
163      and sources for later processing as needed. 
164      """ 
165   
166      __slots__ = ('pre_actions', 
167                   'post_actions', 
168                   'env', 
169                   'overridelist', 
170                   'batches', 
171                   'builder_kw', 
172                   '_memo', 
173                   'lvars', 
174                   '_changed_sources_list', 
175                   '_changed_targets_list', 
176                   '_unchanged_sources_list', 
177                   '_unchanged_targets_list', 
178                   'action_list', 
179                   '_do_execute', 
180                   '_execute_str') 
181   
182 -    def __init__(self, action, env=None, overridelist=[{}], 
183                   targets=[], sources=[], builder_kw={}): 
184          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 
185          self.set_action_list(action) 
186          self.pre_actions = [] 
187          self.post_actions = [] 
188          self.env = env 
189          self.overridelist = overridelist 
190          if targets or sources: 
191              self.batches = [Batch(targets[:], sources[:])] 
192          else: 
193              self.batches = [] 
194          self.builder_kw = builder_kw 
195          self._do_execute = 1 
196          self._execute_str = 1 
197          self._memo = {} 
198   
199 -    def get_lvars(self): 
200          try: 
201              return self.lvars 
202          except AttributeError: 
203              self.lvars = { 
204                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
205                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
206                  'SOURCE' : TSObject(self._get_source), 
207                  'SOURCES' : TSList(self._get_sources), 
208                  'TARGET' : TSObject(self._get_target), 
209                  'TARGETS' : TSList(self._get_targets), 
210                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
211                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
212              } 
213              return self.lvars 
214   
215 -    def _get_changes(self): 
216          cs = [] 
217          ct = [] 
218          us = [] 
219          ut = [] 
220          for b in self.batches: 
221              # don't add targets marked always build to unchanged lists 
222              # add to changed list as they always need to build 
223              if not b.targets[0].always_build and b.targets[0].is_up_to_date(): 
224                  us.extend(list(map(rfile, b.sources))) 
225                  ut.extend(b.targets) 
226              else: 
227                  cs.extend(list(map(rfile, b.sources))) 
228                  ct.extend(b.targets) 
229          self._changed_sources_list = SCons.Util.NodeList(cs) 
230          self._changed_targets_list = SCons.Util.NodeList(ct) 
231          self._unchanged_sources_list = SCons.Util.NodeList(us) 
232          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
233   
234 -    def _get_changed_sources(self, *args, **kw): 
235          try: 
236              return self._changed_sources_list 
237          except AttributeError: 
238              self._get_changes() 
239              return self._changed_sources_list 
240   
241 -    def _get_changed_targets(self, *args, **kw): 
242          try: 
243              return self._changed_targets_list 
244          except AttributeError: 
245              self._get_changes() 
246              return self._changed_targets_list 
247   
248 -    def _get_source(self, *args, **kw): 
249          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
250   
251 -    def _get_sources(self, *args, **kw): 
252          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
253   
254 -    def _get_target(self, *args, **kw): 
255          return self.batches[0].targets[0].get_subst_proxy() 
256   
257 -    def _get_targets(self, *args, **kw): 
258          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
259   
260 -    def _get_unchanged_sources(self, *args, **kw): 
261          try: 
262              return self._unchanged_sources_list 
263          except AttributeError: 
264              self._get_changes() 
265              return self._unchanged_sources_list 
266   
267 -    def _get_unchanged_targets(self, *args, **kw): 
268          try: 
269              return self._unchanged_targets_list 
270          except AttributeError: 
271              self._get_changes() 
272              return self._unchanged_targets_list 
273   
274 -    def get_action_targets(self): 
275          if not self.action_list: 
276              return [] 
277          targets_string = self.action_list[0].get_targets(self.env, self) 
278          if targets_string[0] == '$': 
279              targets_string = targets_string[1:] 
280          return self.get_lvars()[targets_string] 
281   
282 -    def set_action_list(self, action): 
283          import SCons.Util 
284          if not SCons.Util.is_List(action): 
285              if not action: 
286                  import SCons.Errors 
287                  raise SCons.Errors.UserError("Executor must have an action.") 
288              action = [action] 
289          self.action_list = action 
290   
291 -    def get_action_list(self): 
292          if self.action_list is None: 
293              return [] 
294          return self.pre_actions + self.action_list + self.post_actions 
295   
296 -    def get_all_targets(self): 
297          """Returns all targets for all batches of this Executor.""" 
298          result = [] 
299          for batch in self.batches: 
300              result.extend(batch.targets) 
301          return result 
302   
303 -    def get_all_sources(self): 
304          """Returns all sources for all batches of this Executor.""" 
305          result = [] 
306          for batch in self.batches: 
307              result.extend(batch.sources) 
308          return result 
309   
310 -    def get_all_children(self): 
311          """Returns all unique children (dependencies) for all batches 
312          of this Executor. 
313   
314          The Taskmaster can recognize when it's already evaluated a 
315          Node, so we don't have to make this list unique for its intended 
316          canonical use case, but we expect there to be a lot of redundancy 
317          (long lists of batched .cc files #including the same .h files 
318          over and over), so removing the duplicates once up front should 
319          save the Taskmaster a lot of work. 
320          """ 
321          result = SCons.Util.UniqueList([]) 
322          for target in self.get_all_targets(): 
323              result.extend(target.children()) 
324          return result 
325   
326 -    def get_all_prerequisites(self): 
327          """Returns all unique (order-only) prerequisites for all batches 
328          of this Executor. 
329          """ 
330          result = SCons.Util.UniqueList([]) 
331          for target in self.get_all_targets(): 
332              if target.prerequisites is not None: 
333                  result.extend(target.prerequisites) 
334          return result 
335   
336 -    def get_action_side_effects(self): 
337   
338          """Returns all side effects for all batches of this 
339          Executor used by the underlying Action. 
340          """ 
341          result = SCons.Util.UniqueList([]) 
342          for target in self.get_action_targets(): 
343              result.extend(target.side_effects) 
344          return result 
345   
346      @SCons.Memoize.CountMethodCall 
347 -    def get_build_env(self): 
348          """Fetch or create the appropriate build Environment 
349          for this Executor. 
350          """ 
351          try: 
352              return self._memo['get_build_env'] 
353          except KeyError: 
354              pass 
355   
356          # Create the build environment instance with appropriate 
357          # overrides.  These get evaluated against the current 
358          # environment's construction variables so that users can 
359          # add to existing values by referencing the variable in 
360          # the expansion. 
361          overrides = {} 
362          for odict in self.overridelist: 
363              overrides.update(odict) 
364   
365          import SCons.Defaults 
366          env = self.env or SCons.Defaults.DefaultEnvironment() 
367          build_env = env.Override(overrides) 
368   
369          self._memo['get_build_env'] = build_env 
370   
371          return build_env 
372   
373 -    def get_build_scanner_path(self, scanner): 
374          """Fetch the scanner path for this executor's targets and sources. 
375          """ 
376          env = self.get_build_env() 
377          try: 
378              cwd = self.batches[0].targets[0].cwd 
379          except (IndexError, AttributeError): 
380              cwd = None 
381          return scanner.path(env, cwd, 
382                              self.get_all_targets(), 
383                              self.get_all_sources()) 
384   
385 -    def get_kw(self, kw={}): 
386          result = self.builder_kw.copy() 
387          result.update(kw) 
388          result['executor'] = self 
389          return result 
390   
391      # use extra indirection because with new-style objects (Python 2.2 
392      # and above) we can't override special methods, and nullify() needs 
393      # to be able to do this. 
394   
395 -    def __call__(self, target, **kw): 
396          return _do_execute_map[self._do_execute](self, target, kw) 
397   
398 -    def cleanup(self): 
399          self._memo = {} 
400   
401 -    def add_sources(self, sources): 
402          """Add source files to this Executor's list.  This is necessary 
403          for "multi" Builders that can be called repeatedly to build up 
404          a source file list for a given target.""" 
405          # TODO(batch):  extend to multiple batches 
406          assert (len(self.batches) == 1) 
407          # TODO(batch):  remove duplicates? 
408          sources = [x for x in sources if x not in self.batches[0].sources] 
409          self.batches[0].sources.extend(sources) 
410   
411 -    def get_sources(self): 
412          return self.batches[0].sources 
413   
414 -    def add_batch(self, targets, sources): 
415          """Add pair of associated target and source to this Executor's list. 
416          This is necessary for "batch" Builders that can be called repeatedly 
417          to build up a list of matching target and source files that will be 
418          used in order to update multiple target files at once from multiple 
419          corresponding source files, for tools like MSVC that support it.""" 
420          self.batches.append(Batch(targets, sources)) 
421   
422 -    def prepare(self): 
423          """ 
424          Preparatory checks for whether this Executor can go ahead 
425          and (try to) build its targets. 
426          """ 
427          for s in self.get_all_sources(): 
428              if s.missing(): 
429                  msg = "Source `%s' not found, needed by target `%s'." 
430                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
431   
432 -    def add_pre_action(self, action): 
433          self.pre_actions.append(action) 
434   
435 -    def add_post_action(self, action): 
436          self.post_actions.append(action) 
437   
438      # another extra indirection for new-style objects and nullify... 
439   
440 -    def __str__(self): 
441          return _execute_str_map[self._execute_str](self) 
442   
443 -    def nullify(self): 
444          self.cleanup() 
445          self._do_execute = 0 
446          self._execute_str = 0 
447   
448      @SCons.Memoize.CountMethodCall 
449 -    def get_contents(self): 
450          """Fetch the signature contents.  This is the main reason this 
451          class exists, so we can compute this once and cache it regardless 
452          of how many target or source Nodes there are. 
453          """ 
454          try: 
455              return self._memo['get_contents'] 
456          except KeyError: 
457              pass 
458          env = self.get_build_env() 
459   
460          action_list = self.get_action_list() 
461          all_targets = self.get_all_targets() 
462          all_sources = self.get_all_sources() 
463   
464          result = bytearray("",'utf-8').join([action.get_contents(all_targets, 
465                                                                   all_sources, 
466                                                                   env) 
467                                               for action in action_list]) 
468   
469          self._memo['get_contents'] = result 
470          return result 
471   
472 -    def get_timestamp(self): 
473          """Fetch a time stamp for this Executor.  We don't have one, of 
474          course (only files do), but this is the interface used by the 
475          timestamp module. 
476          """ 
477          return 0 
478   
479 -    def scan_targets(self, scanner): 
480          # TODO(batch):  scan by batches 
481          self.scan(scanner, self.get_all_targets()) 
482   
483 -    def scan_sources(self, scanner): 
484          # TODO(batch):  scan by batches 
485          if self.batches[0].sources: 
486              self.scan(scanner, self.get_all_sources()) 
487   
488 -    def scan(self, scanner, node_list): 
489          """Scan a list of this Executor's files (targets or sources) for 
490          implicit dependencies and update all of the targets with them. 
491          This essentially short-circuits an N*M scan of the sources for 
492          each individual target, which is a hell of a lot more efficient. 
493          """ 
494          env = self.get_build_env() 
495          path = self.get_build_scanner_path 
496          kw = self.get_kw() 
497   
498          # TODO(batch):  scan by batches) 
499          deps = [] 
500   
501          for node in node_list: 
502              node.disambiguate() 
503              deps.extend(node.get_implicit_deps(env, scanner, path, kw)) 
504   
505          deps.extend(self.get_implicit_deps()) 
506   
507          for tgt in self.get_all_targets(): 
508              tgt.add_to_implicit(deps) 
509   
510 -    def _get_unignored_sources_key(self, node, ignore=()): 
511          return (node,) + tuple(ignore) 
512   
513      @SCons.Memoize.CountDictCall(_get_unignored_sources_key) 
514 -    def get_unignored_sources(self, node, ignore=()): 
515          key = (node,) + tuple(ignore) 
516          try: 
517              memo_dict = self._memo['get_unignored_sources'] 
518          except KeyError: 
519              memo_dict = {} 
520              self._memo['get_unignored_sources'] = memo_dict 
521          else: 
522              try: 
523                  return memo_dict[key] 
524              except KeyError: 
525                  pass 
526   
527          if node: 
528              # TODO:  better way to do this (it's a linear search, 
529              # but it may not be critical path)? 
530              sourcelist = [] 
531              for b in self.batches: 
532                  if node in b.targets: 
533                      sourcelist = b.sources 
534                      break 
535          else: 
536              sourcelist = self.get_all_sources() 
537          if ignore: 
538              idict = {} 
539              for i in ignore: 
540                  idict[i] = 1 
541              sourcelist = [s for s in sourcelist if s not in idict] 
542   
543          memo_dict[key] = sourcelist 
544   
545          return sourcelist 
546   
547 -    def get_implicit_deps(self): 
548          """Return the executor's implicit dependencies, i.e. the nodes of 
549          the commands to be executed.""" 
550          result = [] 
551          build_env = self.get_build_env() 
552          for act in self.get_action_list(): 
553              deps = act.get_implicit_deps(self.get_all_targets(), 
554                                           self.get_all_sources(), 
555                                           build_env) 
556              result.extend(deps) 
557          return result 
558   
559   
560   
561  _batch_executors = {} 
562   
563 -def GetBatchExecutor(key): 
564      return _batch_executors[key] 
565   
566 -def AddBatchExecutor(key, executor): 
567      assert key not in _batch_executors 
568      _batch_executors[key] = executor 
569   
570  nullenv = None 
571   
572   
573  import SCons.Util 
574 -class NullEnvironment(SCons.Util.Null): 
575      import SCons.CacheDir 
576      _CacheDir_path = None 
577      _CacheDir = SCons.CacheDir.CacheDir(None) 
578 -    def get_CacheDir(self): 
579          return self._CacheDir 
580   
581   
582 -def get_NullEnvironment(): 
583      """Use singleton pattern for Null Environments.""" 
584      global nullenv 
585   
586      if nullenv is None: 
587          nullenv = NullEnvironment() 
588      return nullenv 
589   
590 -class Null(object, with_metaclass(NoSlotsPyPy)): 
591      """A null Executor, with a null build Environment, that does 
592      nothing when the rest of the methods call it. 
593   
594      This might be able to disappear when we refactor things to 
595      disassociate Builders from Nodes entirely, so we're not 
596      going to worry about unit tests for this--at least for now. 
597      """ 
598       
599      __slots__ = ('pre_actions', 
600                   'post_actions', 
601                   'env', 
602                   'overridelist', 
603                   'batches', 
604                   'builder_kw', 
605                   '_memo', 
606                   'lvars', 
607                   '_changed_sources_list', 
608                   '_changed_targets_list', 
609                   '_unchanged_sources_list', 
610                   '_unchanged_targets_list', 
611                   'action_list', 
612                   '_do_execute', 
613                   '_execute_str') 
614       
615 -    def __init__(self, *args, **kw): 
616          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 
617          self.batches = [Batch(kw['targets'][:], [])] 
618 -    def get_build_env(self): 
619          return get_NullEnvironment() 
620 -    def get_build_scanner_path(self): 
621          return None 
622 -    def cleanup(self): 
623          pass 
624 -    def prepare(self): 
625          pass 
626 -    def get_unignored_sources(self, *args, **kw): 
627          return tuple(()) 
628 -    def get_action_targets(self): 
629          return [] 
630 -    def get_action_list(self): 
631          return [] 
632 -    def get_all_targets(self): 
633          return self.batches[0].targets 
634 -    def get_all_sources(self): 
635          return self.batches[0].targets[0].sources 
636 -    def get_all_children(self): 
637          return self.batches[0].targets[0].children() 
638 -    def get_all_prerequisites(self): 
639          return [] 
640 -    def get_action_side_effects(self): 
641          return [] 
642 -    def __call__(self, *args, **kw): 
643          return 0 
644 -    def get_contents(self): 
645          return '' 
646 -    def _morph(self): 
647          """Morph this Null executor to a real Executor object.""" 
648          batches = self.batches 
649          self.__class__ = Executor 
650          self.__init__([])             
651          self.batches = batches 
652   
653      # The following methods require morphing this Null Executor to a 
654      # real Executor object. 
655   
656 -    def add_pre_action(self, action): 
657          self._morph() 
658          self.add_pre_action(action) 
659 -    def add_post_action(self, action): 
660          self._morph() 
661          self.add_post_action(action) 
662 -    def set_action_list(self, action): 
663          self._morph() 
664          self.set_action_list(action) 
665   
666  # Local Variables: 
667  # tab-width:4 
668  # indent-tabs-mode:nil 
669  # End: 
670  # vim: set expandtab tabstop=4 shiftwidth=4: 
671