Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2014 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29   
 30  __revision__ = "src/engine/SCons/Executor.py  2014/09/27 12:51:43 garyo" 
 31   
 32  import collections 
 33   
 34  import SCons.Debug 
 35  from SCons.Debug import logInstanceCreation 
 36  import SCons.Errors 
 37  import SCons.Memoize 
 38   
 39   
 40 -class Batch(object): 
 41      """Remembers exact association between targets 
 42      and sources of executor.""" 
 43 -    def __init__(self, targets=[], sources=[]): 
 44          self.targets = targets 
 45          self.sources = sources 
 46   
 47   
 48   
 49 -class TSList(collections.UserList): 
 50      """A class that implements $TARGETS or $SOURCES expansions by wrapping 
 51      an executor Method.  This class is used in the Executor.lvars() 
 52      to delay creation of NodeList objects until they're needed. 
 53   
 54      Note that we subclass collections.UserList purely so that the 
 55      is_Sequence() function will identify an object of this class as 
 56      a list during variable expansion.  We're not really using any 
 57      collections.UserList methods in practice. 
 58      """ 
 59 -    def __init__(self, func): 
 60          self.func = func 
 61 -    def __getattr__(self, attr): 
 62          nl = self.func() 
 63          return getattr(nl, attr) 
 64 -    def __getitem__(self, i): 
 65          nl = self.func() 
 66          return nl[i] 
 67 -    def __getslice__(self, i, j): 
 68          nl = self.func() 
 69          i = max(i, 0); j = max(j, 0) 
 70          return nl[i:j] 
 71 -    def __str__(self): 
 72          nl = self.func() 
 73          return str(nl) 
 74 -    def __repr__(self): 
 75          nl = self.func() 
 76          return repr(nl) 
 77   
 78 -class TSObject(object): 
 79      """A class that implements $TARGET or $SOURCE expansions by wrapping 
 80      an Executor method. 
 81      """ 
 82 -    def __init__(self, func): 
 83          self.func = func 
 84 -    def __getattr__(self, attr): 
 85          n = self.func() 
 86          return getattr(n, attr) 
 87 -    def __str__(self): 
 88          n = self.func() 
 89          if n: 
 90              return str(n) 
 91          return '' 
 92 -    def __repr__(self): 
 93          n = self.func() 
 94          if n: 
 95              return repr(n) 
 96          return '' 
 97   
 98 -def rfile(node): 
 99      """ 
100      A function to return the results of a Node's rfile() method, 
101      if it exists, and the Node itself otherwise (if it's a Value 
102      Node, e.g.). 
103      """ 
104      try: 
105          rfile = node.rfile 
106      except AttributeError: 
107          return node 
108      else: 
109          return rfile() 
110   
111   
112 -class Executor(object): 
113      """A class for controlling instances of executing an action. 
114   
115      This largely exists to hold a single association of an action, 
116      environment, list of environment override dictionaries, targets 
117      and sources for later processing as needed. 
118      """ 
119   
120      if SCons.Memoize.use_memoizer: 
121          __metaclass__ = SCons.Memoize.Memoized_Metaclass 
122   
123      memoizer_counters = [] 
124   
125 -    def __init__(self, action, env=None, overridelist=[{}], 
126                   targets=[], sources=[], builder_kw={}): 
127          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 
128          self.set_action_list(action) 
129          self.pre_actions = [] 
130          self.post_actions = [] 
131          self.env = env 
132          self.overridelist = overridelist 
133          if targets or sources: 
134              self.batches = [Batch(targets[:], sources[:])] 
135          else: 
136              self.batches = [] 
137          self.builder_kw = builder_kw 
138          self._memo = {} 
139   
140 -    def get_lvars(self): 
141          try: 
142              return self.lvars 
143          except AttributeError: 
144              self.lvars = { 
145                  'CHANGED_SOURCES' : TSList(self._get_changed_sources), 
146                  'CHANGED_TARGETS' : TSList(self._get_changed_targets), 
147                  'SOURCE' : TSObject(self._get_source), 
148                  'SOURCES' : TSList(self._get_sources), 
149                  'TARGET' : TSObject(self._get_target), 
150                  'TARGETS' : TSList(self._get_targets), 
151                  'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 
152                  'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 
153              } 
154              return self.lvars 
155   
156 -    def _get_changes(self): 
157          cs = [] 
158          ct = [] 
159          us = [] 
160          ut = [] 
161          for b in self.batches: 
162              if b.targets[0].is_up_to_date(): 
163                  us.extend(list(map(rfile, b.sources))) 
164                  ut.extend(b.targets) 
165              else: 
166                  cs.extend(list(map(rfile, b.sources))) 
167                  ct.extend(b.targets) 
168          self._changed_sources_list = SCons.Util.NodeList(cs) 
169          self._changed_targets_list = SCons.Util.NodeList(ct) 
170          self._unchanged_sources_list = SCons.Util.NodeList(us) 
171          self._unchanged_targets_list = SCons.Util.NodeList(ut) 
172   
173 -    def _get_changed_sources(self, *args, **kw): 
174          try: 
175              return self._changed_sources_list 
176          except AttributeError: 
177              self._get_changes() 
178              return self._changed_sources_list 
179   
180 -    def _get_changed_targets(self, *args, **kw): 
181          try: 
182              return self._changed_targets_list 
183          except AttributeError: 
184              self._get_changes() 
185              return self._changed_targets_list 
186   
187 -    def _get_source(self, *args, **kw): 
188          #return SCons.Util.NodeList([rfile(self.batches[0].sources[0]).get_subst_proxy()]) 
189          return rfile(self.batches[0].sources[0]).get_subst_proxy() 
190   
191 -    def _get_sources(self, *args, **kw): 
192          return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()]) 
193   
194 -    def _get_target(self, *args, **kw): 
195          #return SCons.Util.NodeList([self.batches[0].targets[0].get_subst_proxy()]) 
196          return self.batches[0].targets[0].get_subst_proxy() 
197   
198 -    def _get_targets(self, *args, **kw): 
199          return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()]) 
200   
201 -    def _get_unchanged_sources(self, *args, **kw): 
202          try: 
203              return self._unchanged_sources_list 
204          except AttributeError: 
205              self._get_changes() 
206              return self._unchanged_sources_list 
207   
208 -    def _get_unchanged_targets(self, *args, **kw): 
209          try: 
210              return self._unchanged_targets_list 
211          except AttributeError: 
212              self._get_changes() 
213              return self._unchanged_targets_list 
214   
215 -    def get_action_targets(self): 
216          if not self.action_list: 
217              return [] 
218          targets_string = self.action_list[0].get_targets(self.env, self) 
219          if targets_string[0] == '$': 
220              targets_string = targets_string[1:] 
221          return self.get_lvars()[targets_string] 
222   
223 -    def set_action_list(self, action): 
224          import SCons.Util 
225          if not SCons.Util.is_List(action): 
226              if not action: 
227                  import SCons.Errors 
228                  raise SCons.Errors.UserError("Executor must have an action.") 
229              action = [action] 
230          self.action_list = action 
231   
232 -    def get_action_list(self): 
233          if self.action_list is None: 
234              return [] 
235          return self.pre_actions + self.action_list + self.post_actions 
236   
237 -    def get_all_targets(self): 
238          """Returns all targets for all batches of this Executor.""" 
239          result = [] 
240          for batch in self.batches: 
241              result.extend(batch.targets) 
242          return result 
243   
244 -    def get_all_sources(self): 
245          """Returns all sources for all batches of this Executor.""" 
246          result = [] 
247          for batch in self.batches: 
248              result.extend(batch.sources) 
249          return result 
250   
251 -    def get_all_children(self): 
252          """Returns all unique children (dependencies) for all batches 
253          of this Executor. 
254   
255          The Taskmaster can recognize when it's already evaluated a 
256          Node, so we don't have to make this list unique for its intended 
257          canonical use case, but we expect there to be a lot of redundancy 
258          (long lists of batched .cc files #including the same .h files 
259          over and over), so removing the duplicates once up front should 
260          save the Taskmaster a lot of work. 
261          """ 
262          result = SCons.Util.UniqueList([]) 
263          for target in self.get_all_targets(): 
264              result.extend(target.children()) 
265          return result 
266   
267 -    def get_all_prerequisites(self): 
268          """Returns all unique (order-only) prerequisites for all batches 
269          of this Executor. 
270          """ 
271          result = SCons.Util.UniqueList([]) 
272          for target in self.get_all_targets(): 
273              if target.prerequisites is not None: 
274                  result.extend(target.prerequisites) 
275          return result 
276   
277 -    def get_action_side_effects(self): 
278   
279          """Returns all side effects for all batches of this 
280          Executor used by the underlying Action. 
281          """ 
282          result = SCons.Util.UniqueList([]) 
283          for target in self.get_action_targets(): 
284              result.extend(target.side_effects) 
285          return result 
286   
287      memoizer_counters.append(SCons.Memoize.CountValue('get_build_env')) 
288   
289 -    def get_build_env(self): 
290          """Fetch or create the appropriate build Environment 
291          for this Executor. 
292          """ 
293          try: 
294              return self._memo['get_build_env'] 
295          except KeyError: 
296              pass 
297   
298          # Create the build environment instance with appropriate 
299          # overrides.  These get evaluated against the current 
300          # environment's construction variables so that users can 
301          # add to existing values by referencing the variable in 
302          # the expansion. 
303          overrides = {} 
304          for odict in self.overridelist: 
305              overrides.update(odict) 
306   
307          import SCons.Defaults 
308          env = self.env or SCons.Defaults.DefaultEnvironment() 
309          build_env = env.Override(overrides) 
310   
311          self._memo['get_build_env'] = build_env 
312   
313          return build_env 
314   
315 -    def get_build_scanner_path(self, scanner): 
316          """Fetch the scanner path for this executor's targets and sources. 
317          """ 
318          env = self.get_build_env() 
319          try: 
320              cwd = self.batches[0].targets[0].cwd 
321          except (IndexError, AttributeError): 
322              cwd = None 
323          return scanner.path(env, cwd, 
324                              self.get_all_targets(), 
325                              self.get_all_sources()) 
326   
327 -    def get_kw(self, kw={}): 
328          result = self.builder_kw.copy() 
329          result.update(kw) 
330          result['executor'] = self 
331          return result 
332   
333 -    def do_nothing(self, target, kw): 
334          return 0 
335   
336 -    def do_execute(self, target, kw): 
337          """Actually execute the action list.""" 
338          env = self.get_build_env() 
339          kw = self.get_kw(kw) 
340          status = 0 
341          for act in self.get_action_list(): 
342              #args = (self.get_all_targets(), self.get_all_sources(), env) 
343              args = ([], [], env) 
344              status = act(*args, **kw) 
345              if isinstance(status, SCons.Errors.BuildError): 
346                  status.executor = self 
347                  raise status 
348              elif status: 
349                  msg = "Error %s" % status 
350                  raise SCons.Errors.BuildError( 
351                      errstr=msg,  
352                      node=self.batches[0].targets, 
353                      executor=self,  
354                      action=act) 
355          return status 
356   
357      # use extra indirection because with new-style objects (Python 2.2 
358      # and above) we can't override special methods, and nullify() needs 
359      # to be able to do this. 
360   
361 -    def __call__(self, target, **kw): 
362          return self.do_execute(target, kw) 
363   
364 -    def cleanup(self): 
365          self._memo = {} 
366   
367 -    def add_sources(self, sources): 
368          """Add source files to this Executor's list.  This is necessary 
369          for "multi" Builders that can be called repeatedly to build up 
370          a source file list for a given target.""" 
371          # TODO(batch):  extend to multiple batches 
372          assert (len(self.batches) == 1) 
373          # TODO(batch):  remove duplicates? 
374          sources = [x for x in sources if x not in self.batches[0].sources] 
375          self.batches[0].sources.extend(sources) 
376   
377 -    def get_sources(self): 
378          return self.batches[0].sources 
379   
380 -    def add_batch(self, targets, sources): 
381          """Add pair of associated target and source to this Executor's list. 
382          This is necessary for "batch" Builders that can be called repeatedly 
383          to build up a list of matching target and source files that will be 
384          used in order to update multiple target files at once from multiple 
385          corresponding source files, for tools like MSVC that support it.""" 
386          self.batches.append(Batch(targets, sources)) 
387   
388 -    def prepare(self): 
389          """ 
390          Preparatory checks for whether this Executor can go ahead 
391          and (try to) build its targets. 
392          """ 
393          for s in self.get_all_sources(): 
394              if s.missing(): 
395                  msg = "Source `%s' not found, needed by target `%s'." 
396                  raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) 
397   
398 -    def add_pre_action(self, action): 
399          self.pre_actions.append(action) 
400   
401 -    def add_post_action(self, action): 
402          self.post_actions.append(action) 
403   
404      # another extra indirection for new-style objects and nullify... 
405   
406 -    def my_str(self): 
407          env = self.get_build_env() 
408          return "\n".join([action.genstring(self.get_all_targets(), 
409                                             self.get_all_sources(), 
410                                             env) 
411                            for action in self.get_action_list()]) 
412   
413   
414 -    def __str__(self): 
415          return self.my_str() 
416   
417 -    def nullify(self): 
418          self.cleanup() 
419          self.do_execute = self.do_nothing 
420          self.my_str     = lambda: '' 
421   
422      memoizer_counters.append(SCons.Memoize.CountValue('get_contents')) 
423   
424 -    def get_contents(self): 
425          """Fetch the signature contents.  This is the main reason this 
426          class exists, so we can compute this once and cache it regardless 
427          of how many target or source Nodes there are. 
428          """ 
429          try: 
430              return self._memo['get_contents'] 
431          except KeyError: 
432              pass 
433          env = self.get_build_env() 
434          result = "".join([action.get_contents(self.get_all_targets(), 
435                                                self.get_all_sources(), 
436                                                env) 
437                            for action in self.get_action_list()]) 
438          self._memo['get_contents'] = result 
439          return result 
440   
441 -    def get_timestamp(self): 
442          """Fetch a time stamp for this Executor.  We don't have one, of 
443          course (only files do), but this is the interface used by the 
444          timestamp module. 
445          """ 
446          return 0 
447   
448 -    def scan_targets(self, scanner): 
449          # TODO(batch):  scan by batches 
450          self.scan(scanner, self.get_all_targets()) 
451   
452 -    def scan_sources(self, scanner): 
453          # TODO(batch):  scan by batches 
454          if self.batches[0].sources: 
455              self.scan(scanner, self.get_all_sources()) 
456   
457 -    def scan(self, scanner, node_list): 
458          """Scan a list of this Executor's files (targets or sources) for 
459          implicit dependencies and update all of the targets with them. 
460          This essentially short-circuits an N*M scan of the sources for 
461          each individual target, which is a hell of a lot more efficient. 
462          """ 
463          env = self.get_build_env() 
464   
465          # TODO(batch):  scan by batches) 
466          deps = [] 
467          if scanner: 
468              for node in node_list: 
469                  node.disambiguate() 
470                  s = scanner.select(node) 
471                  if not s: 
472                      continue 
473                  path = self.get_build_scanner_path(s) 
474                  deps.extend(node.get_implicit_deps(env, s, path)) 
475          else: 
476              kw = self.get_kw() 
477              for node in node_list: 
478                  node.disambiguate() 
479                  scanner = node.get_env_scanner(env, kw) 
480                  if not scanner: 
481                      continue 
482                  scanner = scanner.select(node) 
483                  if not scanner: 
484                      continue 
485                  path = self.get_build_scanner_path(scanner) 
486                  deps.extend(node.get_implicit_deps(env, scanner, path)) 
487   
488          deps.extend(self.get_implicit_deps()) 
489   
490          for tgt in self.get_all_targets(): 
491              tgt.add_to_implicit(deps) 
492   
493 -    def _get_unignored_sources_key(self, node, ignore=()): 
494          return (node,) + tuple(ignore) 
495   
496      memoizer_counters.append(SCons.Memoize.CountDict('get_unignored_sources', _get_unignored_sources_key)) 
497   
498 -    def get_unignored_sources(self, node, ignore=()): 
499          key = (node,) + tuple(ignore) 
500          try: 
501              memo_dict = self._memo['get_unignored_sources'] 
502          except KeyError: 
503              memo_dict = {} 
504              self._memo['get_unignored_sources'] = memo_dict 
505          else: 
506              try: 
507                  return memo_dict[key] 
508              except KeyError: 
509                  pass 
510   
511          if node: 
512              # TODO:  better way to do this (it's a linear search, 
513              # but it may not be critical path)? 
514              sourcelist = [] 
515              for b in self.batches: 
516                  if node in b.targets: 
517                      sourcelist = b.sources 
518                      break 
519          else: 
520              sourcelist = self.get_all_sources() 
521          if ignore: 
522              idict = {} 
523              for i in ignore: 
524                  idict[i] = 1 
525              sourcelist = [s for s in sourcelist if s not in idict] 
526   
527          memo_dict[key] = sourcelist 
528   
529          return sourcelist 
530   
531 -    def get_implicit_deps(self): 
532          """Return the executor's implicit dependencies, i.e. the nodes of 
533          the commands to be executed.""" 
534          result = [] 
535          build_env = self.get_build_env() 
536          for act in self.get_action_list(): 
537              deps = act.get_implicit_deps(self.get_all_targets(), 
538                                           self.get_all_sources(), 
539                                           build_env) 
540              result.extend(deps) 
541          return result 
542   
543   
544   
545  _batch_executors = {} 
546   
547 -def GetBatchExecutor(key): 
548      return _batch_executors[key] 
549   
550 -def AddBatchExecutor(key, executor): 
551      assert key not in _batch_executors 
552      _batch_executors[key] = executor 
553   
554  nullenv = None 
555   
556   
557 -def get_NullEnvironment(): 
558      """Use singleton pattern for Null Environments.""" 
559      global nullenv 
560   
561      import SCons.Util 
562      class NullEnvironment(SCons.Util.Null): 
563          import SCons.CacheDir 
564          _CacheDir_path = None 
565          _CacheDir = SCons.CacheDir.CacheDir(None) 
566          def get_CacheDir(self): 
567              return self._CacheDir 
568   
569      if not nullenv: 
570          nullenv = NullEnvironment() 
571      return nullenv 
572   
573 -class Null(object): 
574      """A null Executor, with a null build Environment, that does 
575      nothing when the rest of the methods call it. 
576   
577      This might be able to disappear when we refactor things to 
578      disassociate Builders from Nodes entirely, so we're not 
579      going to worry about unit tests for this--at least for now. 
580      """ 
581 -    def __init__(self, *args, **kw): 
582          if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 
583          self.batches = [Batch(kw['targets'][:], [])] 
584 -    def get_build_env(self): 
585          return get_NullEnvironment() 
586 -    def get_build_scanner_path(self): 
587          return None 
588 -    def cleanup(self): 
589          pass 
590 -    def prepare(self): 
591          pass 
592 -    def get_unignored_sources(self, *args, **kw): 
593          return tuple(()) 
594 -    def get_action_targets(self): 
595          return [] 
596 -    def get_action_list(self): 
597          return [] 
598 -    def get_all_targets(self): 
599          return self.batches[0].targets 
600 -    def get_all_sources(self): 
601          return self.batches[0].targets[0].sources 
602 -    def get_all_children(self): 
603          return self.batches[0].targets[0].children() 
604 -    def get_all_prerequisites(self): 
605          return [] 
606 -    def get_action_side_effects(self): 
607          return [] 
608 -    def __call__(self, *args, **kw): 
609          return 0 
610 -    def get_contents(self): 
611          return '' 
612 -    def _morph(self): 
613          """Morph this Null executor to a real Executor object.""" 
614          batches = self.batches 
615          self.__class__ = Executor 
616          self.__init__([])             
617          self.batches = batches 
618   
619      # The following methods require morphing this Null Executor to a 
620      # real Executor object. 
621   
622 -    def add_pre_action(self, action): 
623          self._morph() 
624          self.add_pre_action(action) 
625 -    def add_post_action(self, action): 
626          self._morph() 
627          self.add_post_action(action) 
628 -    def set_action_list(self, action): 
629          self._morph() 
630          self.set_action_list(action) 
631   
632  # Local Variables: 
633  # tab-width:4 
634  # indent-tabs-mode:nil 
635  # End: 
636  # vim: set expandtab tabstop=4 shiftwidth=4: 
637