Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2019 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29  from __future__ import print_function 
 30   
 31  __revision__ = "src/engine/SCons/Executor.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" 
 32   
 33  import collections 
 34   
 35  import SCons.Debug 
 36  from SCons.Debug import logInstanceCreation 
 37  import SCons.Errors 
 38  import SCons.Memoize 
 39  import SCons.Util 
 40  from SCons.compat import with_metaclass, NoSlotsPyPy 
 41   
 42 -class Batch(object): 
 43      """Remembers exact association between targets 
 44      and sources of executor.""" 
 45   
 46      __slots__ = ('targets', 
 47                   'sources') 
 48   
 49 -    def __init__(self, targets=[], sources=[]): 
 50          self.targets = targets 
 51          self.sources = sources 
 52   
 53   
 54   
 55 -class TSList(collections.UserList): 
 56      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 57      an executor Method.  This class is used in the Executor.lvars() 
 58      to delay creation of NodeList objects until they're needed. 
 59   
 60      Note that we subclass collections.UserList purely so that the 
 61      is_Sequence() function will identify an object of this class as 
 62      a list during variable expansion.  We're not really using any 
 63      collections.UserList methods in practice. 
 64      """ 
 65 -    def __init__(self, func): 
 66          self.func = func 
 67 -    def __getattr__(self, attr): 
 68          nl = self.func() 
 69          return getattr(nl, attr) 
 70 -    def __getitem__(self, i): 
 71          nl = self.func() 
 72          return nl[i] 
 73 -    def __getslice__(self, i, j): 
 74          nl = self.func() 
 75          i, j = max(i, 0), max(j, 0) 
 76          return nl[i:j] 
 77 -    def __str__(self): 
 78          nl = self.func() 
 79          return str(nl) 
 80 -    def __repr__(self): 
 81          nl = self.func() 
 82          return repr(nl) 
 83   
 84 -class TSObject(object): 
 85      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 86      an Executor method. 
 87      """ 
 88 -    def __init__(self, func): 
 89          self.func = func 
 90 -    def __getattr__(self, attr): 
 91          n = self.func() 
 92          return getattr(n, attr) 
 93 -    def __str__(self): 
 94          n = self.func() 
 95          if n: 
 96              return str(n) 
 97          return '' 
 98 -    def __repr__(self): 
 99          n = self.func() 
100          if n: 
101              return repr(n) 
102          return '' 
103   
104 -def rfile(node): 
105      """ 
106      A function to return the results of a Node's rfile() method, 
107      if it exists, and the Node itself otherwise (if it's a Value 
108      Node, e.g.). 
109      """ 
110      try: 
111          rfile = node.rfile 
112      except AttributeError: 
113          return node 
114      else: 
115          return rfile() 
116   
117   
118 -def execute_nothing(obj, target, kw): 
119      return 0 
120   
121 -def execute_action_list(obj, target, kw): 
122      """Actually execute the action list.""" 
123      env = obj.get_build_env() 
124      kw = obj.get_kw(kw) 
125      status = 0 
126      for act in obj.get_action_list(): 
127          args = ([], [], env) 
128          status = act(*args, **kw) 
129          if isinstance(status, SCons.Errors.BuildError): 
130              status.executor = obj 
131              raise status    # TODO pylint E0702: raising int not allowed 
132          elif status: 
133              msg = "Error %s" % status 
134              raise SCons.Errors.BuildError( 
135                  errstr=msg, 
136                  node=obj.batches[0].targets, 
137                  executor=obj, 
138                  action=act) 
139      return status 
140   
141  _do_execute_map = {0 : execute_nothing, 
142                     1 : execute_action_list} 
143   
144   
145 -def execute_actions_str(obj): 
146      env = obj.get_build_env() 
147      return "\n".join([action.genstring(obj.get_all_targets(), 
148                                         obj.get_all_sources(), 
149                                         env) 
150                        for action in obj.get_action_list()]) 
151   
152 -def execute_null_str(obj): 
153      return '' 
154   
155  _execute_str_map = {0 : execute_null_str, 
156                      1 : execute_actions_str} 
157   
158   
159 -class Executor(object, with_metaclass(NoSlotsPyPy)): 
160      """A class for controlling instances of executing an action. 
161   
162      This largely exists to hold a single association of an action, 
163      environment, list of environment override dictionaries, targets 
164      and sources for later processing as needed. 
165      """ 
166   
167      __slots__ = ('pre_actions', 
168                   'post_actions', 
169                   'env', 
170                   'overridelist', 
171                   'batches', 
172                   'builder_kw', 
173                   '_memo', 
174                   'lvars', 
175                   '_changed_sources_list', 
176                   '_changed_targets_list', 
177                   '_unchanged_sources_list', 
178                   '_unchanged_targets_list', 
179                   'action_list', 
180                   '_do_execute', 
181                   '_execute_str') 
182   
183 -    def __init__(self, action, env=None, overridelist=[{}], 
184                   targets=[], sources=[], builder_kw={}): 
185          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 
186          self.set_action_list(action) 
187          self.pre_actions = [] 
188          self.post_actions = [] 
189          self.env = env 
190          self.overridelist = overridelist 
191          if targets or sources: 
192              self.batches = [Batch(targets[:], sources[:])] 
193          else: 
194              self.batches = [] 
195          self.builder_kw = builder_kw 
196          self._do_execute = 1 
197          self._execute_str = 1 
198          self._memo = {} 
199   
200 -    def get_lvars(self): 
201          try: 
202              return self.lvars 
203          except AttributeError: 
204              self.lvars = { 
205                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
206                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
207                  'SOURCE' : TSObject(self._get_source), 
208                  'SOURCES' : TSList(self._get_sources), 
209                  'TARGET' : TSObject(self._get_target), 
210                  'TARGETS' : TSList(self._get_targets), 
211                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
212                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
213              } 
214              return self.lvars 
215   
216 -    def _get_changes(self): 
217          cs = [] 
218          ct = [] 
219          us = [] 
220          ut = [] 
221          for b in self.batches: 
222              # don't add targets marked always build to unchanged lists 
223              # add to changed list as they always need to build 
224              if not b.targets[0].always_build and b.targets[0].is_up_to_date(): 
225                  us.extend(list(map(rfile, b.sources))) 
226                  ut.extend(b.targets) 
227              else: 
228                  cs.extend(list(map(rfile, b.sources))) 
229                  ct.extend(b.targets) 
230          self._changed_sources_list = SCons.Util.NodeList(cs) 
231          self._changed_targets_list = SCons.Util.NodeList(ct) 
232          self._unchanged_sources_list = SCons.Util.NodeList(us) 
233          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
234   
235 -    def _get_changed_sources(self, *args, **kw): 
236          try: 
237              return self._changed_sources_list 
238          except AttributeError: 
239              self._get_changes() 
240              return self._changed_sources_list 
241   
242 -    def _get_changed_targets(self, *args, **kw): 
243          try: 
244              return self._changed_targets_list 
245          except AttributeError: 
246              self._get_changes() 
247              return self._changed_targets_list 
248   
249 -    def _get_source(self, *args, **kw): 
250          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
251   
252 -    def _get_sources(self, *args, **kw): 
253          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
254   
255 -    def _get_target(self, *args, **kw): 
256          return self.batches[0].targets[0].get_subst_proxy() 
257   
258 -    def _get_targets(self, *args, **kw): 
259          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
260   
261 -    def _get_unchanged_sources(self, *args, **kw): 
262          try: 
263              return self._unchanged_sources_list 
264          except AttributeError: 
265              self._get_changes() 
266              return self._unchanged_sources_list 
267   
268 -    def _get_unchanged_targets(self, *args, **kw): 
269          try: 
270              return self._unchanged_targets_list 
271          except AttributeError: 
272              self._get_changes() 
273              return self._unchanged_targets_list 
274   
275 -    def get_action_targets(self): 
276          if not self.action_list: 
277              return [] 
278          targets_string = self.action_list[0].get_targets(self.env, self) 
279          if targets_string[0] == '$': 
280              targets_string = targets_string[1:] 
281          return self.get_lvars()[targets_string] 
282   
283 -    def set_action_list(self, action): 
284          import SCons.Util 
285          if not SCons.Util.is_List(action): 
286              if not action: 
287                  import SCons.Errors 
288                  raise SCons.Errors.UserError("Executor must have an action.") 
289              action = [action] 
290          self.action_list = action 
291   
292 -    def get_action_list(self): 
293          if self.action_list is None: 
294              return [] 
295          return self.pre_actions + self.action_list + self.post_actions 
296   
297 -    def get_all_targets(self): 
298          """Returns all targets for all batches of this Executor.""" 
299          result = [] 
300          for batch in self.batches: 
301              result.extend(batch.targets) 
302          return result 
303   
304 -    def get_all_sources(self): 
305          """Returns all sources for all batches of this Executor.""" 
306          result = [] 
307          for batch in self.batches: 
308              result.extend(batch.sources) 
309          return result 
310   
311 -    def get_all_children(self): 
312          """Returns all unique children (dependencies) for all batches 
313          of this Executor. 
314   
315          The Taskmaster can recognize when it's already evaluated a 
316          Node, so we don't have to make this list unique for its intended 
317          canonical use case, but we expect there to be a lot of redundancy 
318          (long lists of batched .cc files #including the same .h files 
319          over and over), so removing the duplicates once up front should 
320          save the Taskmaster a lot of work. 
321          """ 
322          result = SCons.Util.UniqueList([]) 
323          for target in self.get_all_targets(): 
324              result.extend(target.children()) 
325          return result 
326   
327 -    def get_all_prerequisites(self): 
328          """Returns all unique (order-only) prerequisites for all batches 
329          of this Executor. 
330          """ 
331          result = SCons.Util.UniqueList([]) 
332          for target in self.get_all_targets(): 
333              if target.prerequisites is not None: 
334                  result.extend(target.prerequisites) 
335          return result 
336   
337 -    def get_action_side_effects(self): 
338   
339          """Returns all side effects for all batches of this 
340          Executor used by the underlying Action. 
341          """ 
342          result = SCons.Util.UniqueList([]) 
343          for target in self.get_action_targets(): 
344              result.extend(target.side_effects) 
345          return result 
346   
347      @SCons.Memoize.CountMethodCall 
348 -    def get_build_env(self): 
349          """Fetch or create the appropriate build Environment 
350          for this Executor. 
351          """ 
352          try: 
353              return self._memo['get_build_env'] 
354          except KeyError: 
355              pass 
356   
357          # Create the build environment instance with appropriate 
358          # overrides.  These get evaluated against the current 
359          # environment's construction variables so that users can 
360          # add to existing values by referencing the variable in 
361          # the expansion. 
362          overrides = {} 
363          for odict in self.overridelist: 
364              overrides.update(odict) 
365   
366          import SCons.Defaults 
367          env = self.env or SCons.Defaults.DefaultEnvironment() 
368          build_env = env.Override(overrides) 
369   
370          self._memo['get_build_env'] = build_env 
371   
372          return build_env 
373   
374 -    def get_build_scanner_path(self, scanner): 
375          """Fetch the scanner path for this executor's targets and sources. 
376          """ 
377          env = self.get_build_env() 
378          try: 
379              cwd = self.batches[0].targets[0].cwd 
380          except (IndexError, AttributeError): 
381              cwd = None 
382          return scanner.path(env, cwd, 
383                              self.get_all_targets(), 
384                              self.get_all_sources()) 
385   
386 -    def get_kw(self, kw={}): 
387          result = self.builder_kw.copy() 
388          result.update(kw) 
389          result['executor'] = self 
390          return result 
391   
392      # use extra indirection because with new-style objects (Python 2.2 
393      # and above) we can't override special methods, and nullify() needs 
394      # to be able to do this. 
395   
396 -    def __call__(self, target, **kw): 
397          return _do_execute_map[self._do_execute](self, target, kw) 
398   
399 -    def cleanup(self): 
400          self._memo = {} 
401   
402 -    def add_sources(self, sources): 
403          """Add source files to this Executor's list.  This is necessary 
404          for "multi" Builders that can be called repeatedly to build up 
405          a source file list for a given target.""" 
406          # TODO(batch):  extend to multiple batches 
407          assert (len(self.batches) == 1) 
408          # TODO(batch):  remove duplicates? 
409          sources = [x for x in sources if x not in self.batches[0].sources] 
410          self.batches[0].sources.extend(sources) 
411   
412 -    def get_sources(self): 
413          return self.batches[0].sources 
414   
415 -    def add_batch(self, targets, sources): 
416          """Add pair of associated target and source to this Executor's list. 
417          This is necessary for "batch" Builders that can be called repeatedly 
418          to build up a list of matching target and source files that will be 
419          used in order to update multiple target files at once from multiple 
420          corresponding source files, for tools like MSVC that support it.""" 
421          self.batches.append(Batch(targets, sources)) 
422   
423 -    def prepare(self): 
424          """ 
425          Preparatory checks for whether this Executor can go ahead 
426          and (try to) build its targets. 
427          """ 
428          for s in self.get_all_sources(): 
429              if s.missing(): 
430                  msg = "Source `%s' not found, needed by target `%s'." 
431                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
432   
433 -    def add_pre_action(self, action): 
434          self.pre_actions.append(action) 
435   
436 -    def add_post_action(self, action): 
437          self.post_actions.append(action) 
438   
439      # another extra indirection for new-style objects and nullify... 
440   
441 -    def __str__(self): 
442          return _execute_str_map[self._execute_str](self) 
443   
444 -    def nullify(self): 
445          self.cleanup() 
446          self._do_execute = 0 
447          self._execute_str = 0 
448   
449      @SCons.Memoize.CountMethodCall 
450 -    def get_contents(self): 
451          """Fetch the signature contents.  This is the main reason this 
452          class exists, so we can compute this once and cache it regardless 
453          of how many target or source Nodes there are. 
454   
455          Returns bytes 
456          """ 
457          try: 
458              return self._memo['get_contents'] 
459          except KeyError: 
460              pass 
461          env = self.get_build_env() 
462   
463          action_list = self.get_action_list() 
464          all_targets = self.get_all_targets() 
465          all_sources = self.get_all_sources() 
466   
467          result = bytearray("",'utf-8').join([action.get_contents(all_targets, 
468                                                                   all_sources, 
469                                                                   env) 
470                                               for action in action_list]) 
471   
472          self._memo['get_contents'] = result 
473          return result 
474   
475 -    def get_timestamp(self): 
476          """Fetch a time stamp for this Executor.  We don't have one, of 
477          course (only files do), but this is the interface used by the 
478          timestamp module. 
479          """ 
480          return 0 
481   
482 -    def scan_targets(self, scanner): 
483          # TODO(batch):  scan by batches 
484          self.scan(scanner, self.get_all_targets()) 
485   
486 -    def scan_sources(self, scanner): 
487          # TODO(batch):  scan by batches 
488          if self.batches[0].sources: 
489              self.scan(scanner, self.get_all_sources()) 
490   
491 -    def scan(self, scanner, node_list): 
492          """Scan a list of this Executor's files (targets or sources) for 
493          implicit dependencies and update all of the targets with them. 
494          This essentially short-circuits an N*M scan of the sources for 
495          each individual target, which is a hell of a lot more efficient. 
496          """ 
497          env = self.get_build_env() 
498          path = self.get_build_scanner_path 
499          kw = self.get_kw() 
500   
501          # TODO(batch):  scan by batches) 
502          deps = [] 
503   
504          for node in node_list: 
505              node.disambiguate() 
506              deps.extend(node.get_implicit_deps(env, scanner, path, kw)) 
507   
508          deps.extend(self.get_implicit_deps()) 
509   
510          for tgt in self.get_all_targets(): 
511              tgt.add_to_implicit(deps) 
512   
513 -    def _get_unignored_sources_key(self, node, ignore=()): 
514          return (node,) + tuple(ignore) 
515   
516      @SCons.Memoize.CountDictCall(_get_unignored_sources_key) 
517 -    def get_unignored_sources(self, node, ignore=()): 
518          key = (node,) + tuple(ignore) 
519          try: 
520              memo_dict = self._memo['get_unignored_sources'] 
521          except KeyError: 
522              memo_dict = {} 
523              self._memo['get_unignored_sources'] = memo_dict 
524          else: 
525              try: 
526                  return memo_dict[key] 
527              except KeyError: 
528                  pass 
529   
530          if node: 
531              # TODO:  better way to do this (it's a linear search, 
532              # but it may not be critical path)? 
533              sourcelist = [] 
534              for b in self.batches: 
535                  if node in b.targets: 
536                      sourcelist = b.sources 
537                      break 
538          else: 
539              sourcelist = self.get_all_sources() 
540          if ignore: 
541              idict = {} 
542              for i in ignore: 
543                  idict[i] = 1 
544              sourcelist = [s for s in sourcelist if s not in idict] 
545   
546          memo_dict[key] = sourcelist 
547   
548          return sourcelist 
549   
550 -    def get_implicit_deps(self): 
551          """Return the executor's implicit dependencies, i.e. the nodes of 
552          the commands to be executed.""" 
553          result = [] 
554          build_env = self.get_build_env() 
555          for act in self.get_action_list(): 
556              deps = act.get_implicit_deps(self.get_all_targets(), 
557                                           self.get_all_sources(), 
558                                           build_env) 
559              result.extend(deps) 
560          return result 
561   
562   
563   
564  _batch_executors = {} 
565   
566 -def GetBatchExecutor(key): 
567      return _batch_executors[key] 
568   
569 -def AddBatchExecutor(key, executor): 
570      assert key not in _batch_executors 
571      _batch_executors[key] = executor 
572   
573  nullenv = None 
574   
575   
576 -class NullEnvironment(SCons.Util.Null): 
577      import SCons.CacheDir 
578      _CacheDir_path = None 
579      _CacheDir = SCons.CacheDir.CacheDir(None) 
580 -    def get_CacheDir(self): 
581          return self._CacheDir 
582   
583   
584 -def get_NullEnvironment(): 
585      """Use singleton pattern for Null Environments.""" 
586      global nullenv 
587   
588      if nullenv is None: 
589          nullenv = NullEnvironment() 
590      return nullenv 
591   
592 -class Null(object, with_metaclass(NoSlotsPyPy)): 
593      """A null Executor, with a null build Environment, that does 
594      nothing when the rest of the methods call it. 
595   
596      This might be able to disappear when we refactor things to 
597      disassociate Builders from Nodes entirely, so we're not 
598      going to worry about unit tests for this--at least for now. 
599      """ 
600   
601      __slots__ = ('pre_actions', 
602                   'post_actions', 
603                   'env', 
604                   'overridelist', 
605                   'batches', 
606                   'builder_kw', 
607                   '_memo', 
608                   'lvars', 
609                   '_changed_sources_list', 
610                   '_changed_targets_list', 
611                   '_unchanged_sources_list', 
612                   '_unchanged_targets_list', 
613                   'action_list', 
614                   '_do_execute', 
615                   '_execute_str') 
616   
617 -    def __init__(self, *args, **kw): 
618          if SCons.Debug.track_instances: 
619              logInstanceCreation(self, 'Executor.Null') 
620          self.batches = [Batch(kw['targets'][:], [])] 
621 -    def get_build_env(self): 
622          return get_NullEnvironment() 
623 -    def get_build_scanner_path(self): 
624          return None 
625 -    def cleanup(self): 
626          pass 
627 -    def prepare(self): 
628          pass 
629 -    def get_unignored_sources(self, *args, **kw): 
630          return tuple(()) 
631 -    def get_action_targets(self): 
632          return [] 
633 -    def get_action_list(self): 
634          return [] 
635 -    def get_all_targets(self): 
636          return self.batches[0].targets 
637 -    def get_all_sources(self): 
638          return self.batches[0].targets[0].sources 
639 -    def get_all_children(self): 
640          return self.batches[0].targets[0].children() 
641 -    def get_all_prerequisites(self): 
642          return [] 
643 -    def get_action_side_effects(self): 
644          return [] 
645 -    def __call__(self, *args, **kw): 
646          return 0 
647 -    def get_contents(self): 
648          return '' 
649 -    def _morph(self): 
650          """Morph this Null executor to a real Executor object.""" 
651          batches = self.batches 
652          self.__class__ = Executor 
653          self.__init__([]) 
654          self.batches = batches 
655   
656      # The following methods require morphing this Null Executor to a 
657      # real Executor object. 
658   
659 -    def add_pre_action(self, action): 
660          self._morph() 
661          self.add_pre_action(action) 
662 -    def add_post_action(self, action): 
663          self._morph() 
664          self.add_post_action(action) 
665 -    def set_action_list(self, action): 
666          self._morph() 
667          self.set_action_list(action) 
668   
669  # Local Variables: 
670  # tab-width:4 
671  # indent-tabs-mode:nil 
672  # End: 
673  # vim: set expandtab tabstop=4 shiftwidth=4: 
674