Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2019 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29  from __future__ import print_function 
 30   
 31  __revision__ = "src/engine/SCons/Executor.py 3a41ed6b288cee8d085373ad7fa02894e1903864 2019-01-23 17:30:35 bdeegan" 
 32   
 33  import collections 
 34   
 35  import SCons.Debug 
 36  from SCons.Debug import logInstanceCreation 
 37  import SCons.Errors 
 38  import SCons.Memoize 
 39  from SCons.compat import with_metaclass, NoSlotsPyPy 
 40   
 41 -class Batch(object): 
 42      """Remembers exact association between targets 
 43      and sources of executor.""" 
 44       
 45      __slots__ = ('targets', 
 46                   'sources') 
 47       
 48 -    def __init__(self, targets=[], sources=[]): 
 49          self.targets = targets 
 50          self.sources = sources 
 51   
 52   
 53   
 54 -class TSList(collections.UserList): 
 55      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 56      an executor Method.  This class is used in the Executor.lvars() 
 57      to delay creation of NodeList objects until they're needed. 
 58   
 59      Note that we subclass collections.UserList purely so that the 
 60      is_Sequence() function will identify an object of this class as 
 61      a list during variable expansion.  We're not really using any 
 62      collections.UserList methods in practice. 
 63      """ 
 64 -    def __init__(self, func): 
 65          self.func = func 
 66 -    def __getattr__(self, attr): 
 67          nl = self.func() 
 68          return getattr(nl, attr) 
 69 -    def __getitem__(self, i): 
 70          nl = self.func() 
 71          return nl[i] 
 72 -    def __getslice__(self, i, j): 
 73          nl = self.func() 
 74          i = max(i, 0); j = max(j, 0) 
 75          return nl[i:j] 
 76 -    def __str__(self): 
 77          nl = self.func() 
 78          return str(nl) 
 79 -    def __repr__(self): 
 80          nl = self.func() 
 81          return repr(nl) 
 82   
 83 -class TSObject(object): 
 84      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 85      an Executor method. 
 86      """ 
 87 -    def __init__(self, func): 
 88          self.func = func 
 89 -    def __getattr__(self, attr): 
 90          n = self.func() 
 91          return getattr(n, attr) 
 92 -    def __str__(self): 
 93          n = self.func() 
 94          if n: 
 95              return str(n) 
 96          return '' 
 97 -    def __repr__(self): 
 98          n = self.func() 
 99          if n: 
100              return repr(n) 
101          return '' 
102   
103 -def rfile(node): 
104      """ 
105      A function to return the results of a Node's rfile() method, 
106      if it exists, and the Node itself otherwise (if it's a Value 
107      Node, e.g.). 
108      """ 
109      try: 
110          rfile = node.rfile 
111      except AttributeError: 
112          return node 
113      else: 
114          return rfile() 
115   
116   
117 -def execute_nothing(obj, target, kw): 
118      return 0 
119   
120 -def execute_action_list(obj, target, kw): 
121      """Actually execute the action list.""" 
122      env = obj.get_build_env() 
123      kw = obj.get_kw(kw) 
124      status = 0 
125      for act in obj.get_action_list(): 
126          args = ([], [], env) 
127          status = act(*args, **kw) 
128          if isinstance(status, SCons.Errors.BuildError): 
129              status.executor = obj 
130              raise status 
131          elif status: 
132              msg = "Error %s" % status 
133              raise SCons.Errors.BuildError( 
134                  errstr=msg,  
135                  node=obj.batches[0].targets, 
136                  executor=obj,  
137                  action=act) 
138      return status 
139   
140  _do_execute_map = {0 : execute_nothing, 
141                     1 : execute_action_list} 
142   
143   
144 -def execute_actions_str(obj): 
145      env = obj.get_build_env() 
146      return "\n".join([action.genstring(obj.get_all_targets(), 
147                                         obj.get_all_sources(), 
148                                         env) 
149                        for action in obj.get_action_list()]) 
150   
151 -def execute_null_str(obj): 
152      return '' 
153   
154  _execute_str_map = {0 : execute_null_str, 
155                      1 : execute_actions_str} 
156   
157   
158 -class Executor(object, with_metaclass(NoSlotsPyPy)): 
159      """A class for controlling instances of executing an action. 
160   
161      This largely exists to hold a single association of an action, 
162      environment, list of environment override dictionaries, targets 
163      and sources for later processing as needed. 
164      """ 
165   
166      __slots__ = ('pre_actions', 
167                   'post_actions', 
168                   'env', 
169                   'overridelist', 
170                   'batches', 
171                   'builder_kw', 
172                   '_memo', 
173                   'lvars', 
174                   '_changed_sources_list', 
175                   '_changed_targets_list', 
176                   '_unchanged_sources_list', 
177                   '_unchanged_targets_list', 
178                   'action_list', 
179                   '_do_execute', 
180                   '_execute_str') 
181   
182 -    def __init__(self, action, env=None, overridelist=[{}], 
183                   targets=[], sources=[], builder_kw={}): 
184          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 
185          self.set_action_list(action) 
186          self.pre_actions = [] 
187          self.post_actions = [] 
188          self.env = env 
189          self.overridelist = overridelist 
190          if targets or sources: 
191              self.batches = [Batch(targets[:], sources[:])] 
192          else: 
193              self.batches = [] 
194          self.builder_kw = builder_kw 
195          self._do_execute = 1 
196          self._execute_str = 1 
197          self._memo = {} 
198   
199 -    def get_lvars(self): 
200          try: 
201              return self.lvars 
202          except AttributeError: 
203              self.lvars = { 
204                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
205                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
206                  'SOURCE' : TSObject(self._get_source), 
207                  'SOURCES' : TSList(self._get_sources), 
208                  'TARGET' : TSObject(self._get_target), 
209                  'TARGETS' : TSList(self._get_targets), 
210                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
211                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
212              } 
213              return self.lvars 
214   
215 -    def _get_changes(self): 
216          cs = [] 
217          ct = [] 
218          us = [] 
219          ut = [] 
220          for b in self.batches: 
221              # don't add targets marked always build to unchanged lists 
222              # add to changed list as they always need to build 
223              if not b.targets[0].always_build and b.targets[0].is_up_to_date(): 
224                  us.extend(list(map(rfile, b.sources))) 
225                  ut.extend(b.targets) 
226              else: 
227                  cs.extend(list(map(rfile, b.sources))) 
228                  ct.extend(b.targets) 
229          self._changed_sources_list = SCons.Util.NodeList(cs) 
230          self._changed_targets_list = SCons.Util.NodeList(ct) 
231          self._unchanged_sources_list = SCons.Util.NodeList(us) 
232          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
233   
234 -    def _get_changed_sources(self, *args, **kw): 
235          try: 
236              return self._changed_sources_list 
237          except AttributeError: 
238              self._get_changes() 
239              return self._changed_sources_list 
240   
241 -    def _get_changed_targets(self, *args, **kw): 
242          try: 
243              return self._changed_targets_list 
244          except AttributeError: 
245              self._get_changes() 
246              return self._changed_targets_list 
247   
248 -    def _get_source(self, *args, **kw): 
249          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
250   
251 -    def _get_sources(self, *args, **kw): 
252          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
253   
254 -    def _get_target(self, *args, **kw): 
255          return self.batches[0].targets[0].get_subst_proxy() 
256   
257 -    def _get_targets(self, *args, **kw): 
258          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
259   
260 -    def _get_unchanged_sources(self, *args, **kw): 
261          try: 
262              return self._unchanged_sources_list 
263          except AttributeError: 
264              self._get_changes() 
265              return self._unchanged_sources_list 
266   
267 -    def _get_unchanged_targets(self, *args, **kw): 
268          try: 
269              return self._unchanged_targets_list 
270          except AttributeError: 
271              self._get_changes() 
272              return self._unchanged_targets_list 
273   
274 -    def get_action_targets(self): 
275          if not self.action_list: 
276              return [] 
277          targets_string = self.action_list[0].get_targets(self.env, self) 
278          if targets_string[0] == '$': 
279              targets_string = targets_string[1:] 
280          return self.get_lvars()[targets_string] 
281   
282 -    def set_action_list(self, action): 
283          import SCons.Util 
284          if not SCons.Util.is_List(action): 
285              if not action: 
286                  import SCons.Errors 
287                  raise SCons.Errors.UserError("Executor must have an action.") 
288              action = [action] 
289          self.action_list = action 
290   
291 -    def get_action_list(self): 
292          if self.action_list is None: 
293              return [] 
294          return self.pre_actions + self.action_list + self.post_actions 
295   
296 -    def get_all_targets(self): 
297          """Returns all targets for all batches of this Executor.""" 
298          result = [] 
299          for batch in self.batches: 
300              result.extend(batch.targets) 
301          return result 
302   
303 -    def get_all_sources(self): 
304          """Returns all sources for all batches of this Executor.""" 
305          result = [] 
306          for batch in self.batches: 
307              result.extend(batch.sources) 
308          return result 
309   
310 -    def get_all_children(self): 
311          """Returns all unique children (dependencies) for all batches 
312          of this Executor. 
313   
314          The Taskmaster can recognize when it's already evaluated a 
315          Node, so we don't have to make this list unique for its intended 
316          canonical use case, but we expect there to be a lot of redundancy 
317          (long lists of batched .cc files #including the same .h files 
318          over and over), so removing the duplicates once up front should 
319          save the Taskmaster a lot of work. 
320          """ 
321          result = SCons.Util.UniqueList([]) 
322          for target in self.get_all_targets(): 
323              result.extend(target.children()) 
324          return result 
325   
326 -    def get_all_prerequisites(self): 
327          """Returns all unique (order-only) prerequisites for all batches 
328          of this Executor. 
329          """ 
330          result = SCons.Util.UniqueList([]) 
331          for target in self.get_all_targets(): 
332              if target.prerequisites is not None: 
333                  result.extend(target.prerequisites) 
334          return result 
335   
336 -    def get_action_side_effects(self): 
337   
338          """Returns all side effects for all batches of this 
339          Executor used by the underlying Action. 
340          """ 
341          result = SCons.Util.UniqueList([]) 
342          for target in self.get_action_targets(): 
343              result.extend(target.side_effects) 
344          return result 
345   
346      @SCons.Memoize.CountMethodCall 
347 -    def get_build_env(self): 
348          """Fetch or create the appropriate build Environment 
349          for this Executor. 
350          """ 
351          try: 
352              return self._memo['get_build_env'] 
353          except KeyError: 
354              pass 
355   
356          # Create the build environment instance with appropriate 
357          # overrides.  These get evaluated against the current 
358          # environment's construction variables so that users can 
359          # add to existing values by referencing the variable in 
360          # the expansion. 
361          overrides = {} 
362          for odict in self.overridelist: 
363              overrides.update(odict) 
364   
365          import SCons.Defaults 
366          env = self.env or SCons.Defaults.DefaultEnvironment() 
367          build_env = env.Override(overrides) 
368   
369          self._memo['get_build_env'] = build_env 
370   
371          return build_env 
372   
373 -    def get_build_scanner_path(self, scanner): 
374          """Fetch the scanner path for this executor's targets and sources. 
375          """ 
376          env = self.get_build_env() 
377          try: 
378              cwd = self.batches[0].targets[0].cwd 
379          except (IndexError, AttributeError): 
380              cwd = None 
381          return scanner.path(env, cwd, 
382                              self.get_all_targets(), 
383                              self.get_all_sources()) 
384   
385 -    def get_kw(self, kw={}): 
386          result = self.builder_kw.copy() 
387          result.update(kw) 
388          result['executor'] = self 
389          return result 
390   
391      # use extra indirection because with new-style objects (Python 2.2 
392      # and above) we can't override special methods, and nullify() needs 
393      # to be able to do this. 
394   
395 -    def __call__(self, target, **kw): 
396          return _do_execute_map[self._do_execute](self, target, kw) 
397   
398 -    def cleanup(self): 
399          self._memo = {} 
400   
401 -    def add_sources(self, sources): 
402          """Add source files to this Executor's list.  This is necessary 
403          for "multi" Builders that can be called repeatedly to build up 
404          a source file list for a given target.""" 
405          # TODO(batch):  extend to multiple batches 
406          assert (len(self.batches) == 1) 
407          # TODO(batch):  remove duplicates? 
408          sources = [x for x in sources if x not in self.batches[0].sources] 
409          self.batches[0].sources.extend(sources) 
410   
411 -    def get_sources(self): 
412          return self.batches[0].sources 
413   
414 -    def add_batch(self, targets, sources): 
415          """Add pair of associated target and source to this Executor's list. 
416          This is necessary for "batch" Builders that can be called repeatedly 
417          to build up a list of matching target and source files that will be 
418          used in order to update multiple target files at once from multiple 
419          corresponding source files, for tools like MSVC that support it.""" 
420          self.batches.append(Batch(targets, sources)) 
421   
422 -    def prepare(self): 
423          """ 
424          Preparatory checks for whether this Executor can go ahead 
425          and (try to) build its targets. 
426          """ 
427          for s in self.get_all_sources(): 
428              if s.missing(): 
429                  msg = "Source `%s' not found, needed by target `%s'." 
430                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
431   
432 -    def add_pre_action(self, action): 
433          self.pre_actions.append(action) 
434   
435 -    def add_post_action(self, action): 
436          self.post_actions.append(action) 
437   
438      # another extra indirection for new-style objects and nullify... 
439   
440 -    def __str__(self): 
441          return _execute_str_map[self._execute_str](self) 
442   
443 -    def nullify(self): 
444          self.cleanup() 
445          self._do_execute = 0 
446          self._execute_str = 0 
447   
448      @SCons.Memoize.CountMethodCall 
449 -    def get_contents(self): 
450          """Fetch the signature contents.  This is the main reason this 
451          class exists, so we can compute this once and cache it regardless 
452          of how many target or source Nodes there are. 
453   
454          Returns bytes 
455          """ 
456          try: 
457              return self._memo['get_contents'] 
458          except KeyError: 
459              pass 
460          env = self.get_build_env() 
461   
462          action_list = self.get_action_list() 
463          all_targets = self.get_all_targets() 
464          all_sources = self.get_all_sources() 
465   
466          result = bytearray("",'utf-8').join([action.get_contents(all_targets, 
467                                                                   all_sources, 
468                                                                   env) 
469                                               for action in action_list]) 
470   
471          self._memo['get_contents'] = result 
472          return result 
473   
474 -    def get_timestamp(self): 
475          """Fetch a time stamp for this Executor.  We don't have one, of 
476          course (only files do), but this is the interface used by the 
477          timestamp module. 
478          """ 
479          return 0 
480   
481 -    def scan_targets(self, scanner): 
482          # TODO(batch):  scan by batches 
483          self.scan(scanner, self.get_all_targets()) 
484   
485 -    def scan_sources(self, scanner): 
486          # TODO(batch):  scan by batches 
487          if self.batches[0].sources: 
488              self.scan(scanner, self.get_all_sources()) 
489   
490 -    def scan(self, scanner, node_list): 
491          """Scan a list of this Executor's files (targets or sources) for 
492          implicit dependencies and update all of the targets with them. 
493          This essentially short-circuits an N*M scan of the sources for 
494          each individual target, which is a hell of a lot more efficient. 
495          """ 
496          env = self.get_build_env() 
497          path = self.get_build_scanner_path 
498          kw = self.get_kw() 
499   
500          # TODO(batch):  scan by batches) 
501          deps = [] 
502   
503          for node in node_list: 
504              node.disambiguate() 
505              deps.extend(node.get_implicit_deps(env, scanner, path, kw)) 
506   
507          deps.extend(self.get_implicit_deps()) 
508   
509          for tgt in self.get_all_targets(): 
510              tgt.add_to_implicit(deps) 
511   
512 -    def _get_unignored_sources_key(self, node, ignore=()): 
513          return (node,) + tuple(ignore) 
514   
515      @SCons.Memoize.CountDictCall(_get_unignored_sources_key) 
516 -    def get_unignored_sources(self, node, ignore=()): 
517          key = (node,) + tuple(ignore) 
518          try: 
519              memo_dict = self._memo['get_unignored_sources'] 
520          except KeyError: 
521              memo_dict = {} 
522              self._memo['get_unignored_sources'] = memo_dict 
523          else: 
524              try: 
525                  return memo_dict[key] 
526              except KeyError: 
527                  pass 
528   
529          if node: 
530              # TODO:  better way to do this (it's a linear search, 
531              # but it may not be critical path)? 
532              sourcelist = [] 
533              for b in self.batches: 
534                  if node in b.targets: 
535                      sourcelist = b.sources 
536                      break 
537          else: 
538              sourcelist = self.get_all_sources() 
539          if ignore: 
540              idict = {} 
541              for i in ignore: 
542                  idict[i] = 1 
543              sourcelist = [s for s in sourcelist if s not in idict] 
544   
545          memo_dict[key] = sourcelist 
546   
547          return sourcelist 
548   
549 -    def get_implicit_deps(self): 
550          """Return the executor's implicit dependencies, i.e. the nodes of 
551          the commands to be executed.""" 
552          result = [] 
553          build_env = self.get_build_env() 
554          for act in self.get_action_list(): 
555              deps = act.get_implicit_deps(self.get_all_targets(), 
556                                           self.get_all_sources(), 
557                                           build_env) 
558              result.extend(deps) 
559          return result 
560   
561   
562   
563  _batch_executors = {} 
564   
565 -def GetBatchExecutor(key): 
566      return _batch_executors[key] 
567   
568 -def AddBatchExecutor(key, executor): 
569      assert key not in _batch_executors 
570      _batch_executors[key] = executor 
571   
572  nullenv = None 
573   
574   
575  import SCons.Util 
576 -class NullEnvironment(SCons.Util.Null): 
577      import SCons.CacheDir 
578      _CacheDir_path = None 
579      _CacheDir = SCons.CacheDir.CacheDir(None) 
580 -    def get_CacheDir(self): 
581          return self._CacheDir 
582   
583   
584 -def get_NullEnvironment(): 
585      """Use singleton pattern for Null Environments.""" 
586      global nullenv 
587   
588      if nullenv is None: 
589          nullenv = NullEnvironment() 
590      return nullenv 
591   
592 -class Null(object, with_metaclass(NoSlotsPyPy)): 
593      """A null Executor, with a null build Environment, that does 
594      nothing when the rest of the methods call it. 
595   
596      This might be able to disappear when we refactor things to 
597      disassociate Builders from Nodes entirely, so we're not 
598      going to worry about unit tests for this--at least for now. 
599      """ 
600       
601      __slots__ = ('pre_actions', 
602                   'post_actions', 
603                   'env', 
604                   'overridelist', 
605                   'batches', 
606                   'builder_kw', 
607                   '_memo', 
608                   'lvars', 
609                   '_changed_sources_list', 
610                   '_changed_targets_list', 
611                   '_unchanged_sources_list', 
612                   '_unchanged_targets_list', 
613                   'action_list', 
614                   '_do_execute', 
615                   '_execute_str') 
616       
617 -    def __init__(self, *args, **kw): 
618          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 
619          self.batches = [Batch(kw['targets'][:], [])] 
620 -    def get_build_env(self): 
621          return get_NullEnvironment() 
622 -    def get_build_scanner_path(self): 
623          return None 
624 -    def cleanup(self): 
625          pass 
626 -    def prepare(self): 
627          pass 
628 -    def get_unignored_sources(self, *args, **kw): 
629          return tuple(()) 
630 -    def get_action_targets(self): 
631          return [] 
632 -    def get_action_list(self): 
633          return [] 
634 -    def get_all_targets(self): 
635          return self.batches[0].targets 
636 -    def get_all_sources(self): 
637          return self.batches[0].targets[0].sources 
638 -    def get_all_children(self): 
639          return self.batches[0].targets[0].children() 
640 -    def get_all_prerequisites(self): 
641          return [] 
642 -    def get_action_side_effects(self): 
643          return [] 
644 -    def __call__(self, *args, **kw): 
645          return 0 
646 -    def get_contents(self): 
647          return '' 
648 -    def _morph(self): 
649          """Morph this Null executor to a real Executor object.""" 
650          batches = self.batches 
651          self.__class__ = Executor 
652          self.__init__([])             
653          self.batches = batches 
654   
655      # The following methods require morphing this Null Executor to a 
656      # real Executor object. 
657   
658 -    def add_pre_action(self, action): 
659          self._morph() 
660          self.add_pre_action(action) 
661 -    def add_post_action(self, action): 
662          self._morph() 
663          self.add_post_action(action) 
664 -    def set_action_list(self, action): 
665          self._morph() 
666          self.set_action_list(action) 
667   
668  # Local Variables: 
669  # tab-width:4 
670  # indent-tabs-mode:nil 
671  # End: 
672  # vim: set expandtab tabstop=4 shiftwidth=4: 
673