Package SCons :: Module Executor
[hide private]
[frames] | no frames]

Source Code for Module SCons.Executor

  1  """SCons.Executor 
  2   
  3  A module for executing actions with specific lists of target and source 
  4  Nodes. 
  5   
  6  """ 
  7   
  8  # 
  9  # Copyright (c) 2001 - 2019 The SCons Foundation 
 10  # 
 11  # Permission is hereby granted, free of charge, to any person obtaining 
 12  # a copy of this software and associated documentation files (the 
 13  # "Software"), to deal in the Software without restriction, including 
 14  # without limitation the rights to use, copy, modify, merge, publish, 
 15  # distribute, sublicense, and/or sell copies of the Software, and to 
 16  # permit persons to whom the Software is furnished to do so, subject to 
 17  # the following conditions: 
 18  # 
 19  # The above copyright notice and this permission notice shall be included 
 20  # in all copies or substantial portions of the Software. 
 21  # 
 22  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 23  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 24  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 25  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 26  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 27  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 28  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 29  from __future__ import print_function 
 30   
 31  __revision__ = "src/engine/SCons/Executor.py 3a41ed6b288cee8d085373ad7fa02894e1903864 2019-01-23 17:30:35 bdeegan" 
 32   
 33  import collections 
 34   
 35  import SCons.Debug 
 36  from SCons.Debug import logInstanceCreation 
 37  import SCons.Errors 
 38  import SCons.Memoize 
 39  from SCons.compat import with_metaclass, NoSlotsPyPy 
40 41 -class Batch(object):
42 """Remembers exact association between targets 43 and sources of executor.""" 44 45 __slots__ = ('targets', 46 'sources') 47
48 - def __init__(self, targets=[], sources=[]):
49 self.targets = targets 50 self.sources = sources
51
52 53 54 -class TSList(collections.UserList):
55 """A class that implements $TARGETS or $SOURCES expansions by wrapping 56 an executor Method. This class is used in the Executor.lvars() 57 to delay creation of NodeList objects until they're needed. 58 59 Note that we subclass collections.UserList purely so that the 60 is_Sequence() function will identify an object of this class as 61 a list during variable expansion. We're not really using any 62 collections.UserList methods in practice. 63 """
64 - def __init__(self, func):
65 self.func = func
66 - def __getattr__(self, attr):
67 nl = self.func() 68 return getattr(nl, attr)
69 - def __getitem__(self, i):
70 nl = self.func() 71 return nl[i]
72 - def __getslice__(self, i, j):
73 nl = self.func() 74 i = max(i, 0); j = max(j, 0) 75 return nl[i:j]
76 - def __str__(self):
77 nl = self.func() 78 return str(nl)
79 - def __repr__(self):
80 nl = self.func() 81 return repr(nl)
82
83 -class TSObject(object):
84 """A class that implements $TARGET or $SOURCE expansions by wrapping 85 an Executor method. 86 """
87 - def __init__(self, func):
88 self.func = func
89 - def __getattr__(self, attr):
90 n = self.func() 91 return getattr(n, attr)
92 - def __str__(self):
93 n = self.func() 94 if n: 95 return str(n) 96 return ''
97 - def __repr__(self):
98 n = self.func() 99 if n: 100 return repr(n) 101 return ''
102
103 -def rfile(node):
104 """ 105 A function to return the results of a Node's rfile() method, 106 if it exists, and the Node itself otherwise (if it's a Value 107 Node, e.g.). 108 """ 109 try: 110 rfile = node.rfile 111 except AttributeError: 112 return node 113 else: 114 return rfile()
115
116 117 -def execute_nothing(obj, target, kw):
118 return 0
119
120 -def execute_action_list(obj, target, kw):
121 """Actually execute the action list.""" 122 env = obj.get_build_env() 123 kw = obj.get_kw(kw) 124 status = 0 125 for act in obj.get_action_list(): 126 args = ([], [], env) 127 status = act(*args, **kw) 128 if isinstance(status, SCons.Errors.BuildError): 129 status.executor = obj 130 raise status 131 elif status: 132 msg = "Error %s" % status 133 raise SCons.Errors.BuildError( 134 errstr=msg, 135 node=obj.batches[0].targets, 136 executor=obj, 137 action=act) 138 return status
139 140 _do_execute_map = {0 : execute_nothing, 141 1 : execute_action_list}
142 143 144 -def execute_actions_str(obj):
145 env = obj.get_build_env() 146 return "\n".join([action.genstring(obj.get_all_targets(), 147 obj.get_all_sources(), 148 env) 149 for action in obj.get_action_list()])
150
151 -def execute_null_str(obj):
152 return ''
153 154 _execute_str_map = {0 : execute_null_str, 155 1 : execute_actions_str}
156 157 158 -class Executor(object, with_metaclass(NoSlotsPyPy)):
159 """A class for controlling instances of executing an action. 160 161 This largely exists to hold a single association of an action, 162 environment, list of environment override dictionaries, targets 163 and sources for later processing as needed. 164 """ 165 166 __slots__ = ('pre_actions', 167 'post_actions', 168 'env', 169 'overridelist', 170 'batches', 171 'builder_kw', 172 '_memo', 173 'lvars', 174 '_changed_sources_list', 175 '_changed_targets_list', 176 '_unchanged_sources_list', 177 '_unchanged_targets_list', 178 'action_list', 179 '_do_execute', 180 '_execute_str') 181
182 - def __init__(self, action, env=None, overridelist=[{}], 183 targets=[], sources=[], builder_kw={}):
184 if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Executor') 185 self.set_action_list(action) 186 self.pre_actions = [] 187 self.post_actions = [] 188 self.env = env 189 self.overridelist = overridelist 190 if targets or sources: 191 self.batches = [Batch(targets[:], sources[:])] 192 else: 193 self.batches = [] 194 self.builder_kw = builder_kw 195 self._do_execute = 1 196 self._execute_str = 1 197 self._memo = {}
198
199 - def get_lvars(self):
200 try: 201 return self.lvars 202 except AttributeError: 203 self.lvars = { 204 'CHANGED_SOURCES' : TSList(self._get_changed_sources), 205 'CHANGED_TARGETS' : TSList(self._get_changed_targets), 206 'SOURCE' : TSObject(self._get_source), 207 'SOURCES' : TSList(self._get_sources), 208 'TARGET' : TSObject(self._get_target), 209 'TARGETS' : TSList(self._get_targets), 210 'UNCHANGED_SOURCES' : TSList(self._get_unchanged_sources), 211 'UNCHANGED_TARGETS' : TSList(self._get_unchanged_targets), 212 } 213 return self.lvars
214
215 - def _get_changes(self):
216 cs = [] 217 ct = [] 218 us = [] 219 ut = [] 220 for b in self.batches: 221 # don't add targets marked always build to unchanged lists 222 # add to changed list as they always need to build 223 if not b.targets[0].always_build and b.targets[0].is_up_to_date(): 224 us.extend(list(map(rfile, b.sources))) 225 ut.extend(b.targets) 226 else: 227 cs.extend(list(map(rfile, b.sources))) 228 ct.extend(b.targets) 229 self._changed_sources_list = SCons.Util.NodeList(cs) 230 self._changed_targets_list = SCons.Util.NodeList(ct) 231 self._unchanged_sources_list = SCons.Util.NodeList(us) 232 self._unchanged_targets_list = SCons.Util.NodeList(ut)
233
234 - def _get_changed_sources(self, *args, **kw):
235 try: 236 return self._changed_sources_list 237 except AttributeError: 238 self._get_changes() 239 return self._changed_sources_list
240
241 - def _get_changed_targets(self, *args, **kw):
242 try: 243 return self._changed_targets_list 244 except AttributeError: 245 self._get_changes() 246 return self._changed_targets_list
247
248 - def _get_source(self, *args, **kw):
249 return rfile(self.batches[0].sources[0]).get_subst_proxy()
250
251 - def _get_sources(self, *args, **kw):
252 return SCons.Util.NodeList([rfile(n).get_subst_proxy() for n in self.get_all_sources()])
253
254 - def _get_target(self, *args, **kw):
255 return self.batches[0].targets[0].get_subst_proxy()
256
257 - def _get_targets(self, *args, **kw):
258 return SCons.Util.NodeList([n.get_subst_proxy() for n in self.get_all_targets()])
259
260 - def _get_unchanged_sources(self, *args, **kw):
261 try: 262 return self._unchanged_sources_list 263 except AttributeError: 264 self._get_changes() 265 return self._unchanged_sources_list
266
267 - def _get_unchanged_targets(self, *args, **kw):
268 try: 269 return self._unchanged_targets_list 270 except AttributeError: 271 self._get_changes() 272 return self._unchanged_targets_list
273
274 - def get_action_targets(self):
275 if not self.action_list: 276 return [] 277 targets_string = self.action_list[0].get_targets(self.env, self) 278 if targets_string[0] == '$': 279 targets_string = targets_string[1:] 280 return self.get_lvars()[targets_string]
281
282 - def set_action_list(self, action):
283 import SCons.Util 284 if not SCons.Util.is_List(action): 285 if not action: 286 import SCons.Errors 287 raise SCons.Errors.UserError("Executor must have an action.") 288 action = [action] 289 self.action_list = action
290
291 - def get_action_list(self):
292 if self.action_list is None: 293 return [] 294 return self.pre_actions + self.action_list + self.post_actions
295
296 - def get_all_targets(self):
297 """Returns all targets for all batches of this Executor.""" 298 result = [] 299 for batch in self.batches: 300 result.extend(batch.targets) 301 return result
302
303 - def get_all_sources(self):
304 """Returns all sources for all batches of this Executor.""" 305 result = [] 306 for batch in self.batches: 307 result.extend(batch.sources) 308 return result
309
310 - def get_all_children(self):
311 """Returns all unique children (dependencies) for all batches 312 of this Executor. 313 314 The Taskmaster can recognize when it's already evaluated a 315 Node, so we don't have to make this list unique for its intended 316 canonical use case, but we expect there to be a lot of redundancy 317 (long lists of batched .cc files #including the same .h files 318 over and over), so removing the duplicates once up front should 319 save the Taskmaster a lot of work. 320 """ 321 result = SCons.Util.UniqueList([]) 322 for target in self.get_all_targets(): 323 result.extend(target.children()) 324 return result
325
326 - def get_all_prerequisites(self):
327 """Returns all unique (order-only) prerequisites for all batches 328 of this Executor. 329 """ 330 result = SCons.Util.UniqueList([]) 331 for target in self.get_all_targets(): 332 if target.prerequisites is not None: 333 result.extend(target.prerequisites) 334 return result
335
336 - def get_action_side_effects(self):
337 338 """Returns all side effects for all batches of this 339 Executor used by the underlying Action. 340 """ 341 result = SCons.Util.UniqueList([]) 342 for target in self.get_action_targets(): 343 result.extend(target.side_effects) 344 return result
345 346 @SCons.Memoize.CountMethodCall
347 - def get_build_env(self):
348 """Fetch or create the appropriate build Environment 349 for this Executor. 350 """ 351 try: 352 return self._memo['get_build_env'] 353 except KeyError: 354 pass 355 356 # Create the build environment instance with appropriate 357 # overrides. These get evaluated against the current 358 # environment's construction variables so that users can 359 # add to existing values by referencing the variable in 360 # the expansion. 361 overrides = {} 362 for odict in self.overridelist: 363 overrides.update(odict) 364 365 import SCons.Defaults 366 env = self.env or SCons.Defaults.DefaultEnvironment() 367 build_env = env.Override(overrides) 368 369 self._memo['get_build_env'] = build_env 370 371 return build_env
372
373 - def get_build_scanner_path(self, scanner):
374 """Fetch the scanner path for this executor's targets and sources. 375 """ 376 env = self.get_build_env() 377 try: 378 cwd = self.batches[0].targets[0].cwd 379 except (IndexError, AttributeError): 380 cwd = None 381 return scanner.path(env, cwd, 382 self.get_all_targets(), 383 self.get_all_sources())
384
385 - def get_kw(self, kw={}):
386 result = self.builder_kw.copy() 387 result.update(kw) 388 result['executor'] = self 389 return result
390 391 # use extra indirection because with new-style objects (Python 2.2 392 # and above) we can't override special methods, and nullify() needs 393 # to be able to do this. 394
395 - def __call__(self, target, **kw):
396 return _do_execute_map[self._do_execute](self, target, kw)
397
398 - def cleanup(self):
399 self._memo = {}
400
401 - def add_sources(self, sources):
402 """Add source files to this Executor's list. This is necessary 403 for "multi" Builders that can be called repeatedly to build up 404 a source file list for a given target.""" 405 # TODO(batch): extend to multiple batches 406 assert (len(self.batches) == 1) 407 # TODO(batch): remove duplicates? 408 sources = [x for x in sources if x not in self.batches[0].sources] 409 self.batches[0].sources.extend(sources)
410
411 - def get_sources(self):
412 return self.batches[0].sources
413
414 - def add_batch(self, targets, sources):
415 """Add pair of associated target and source to this Executor's list. 416 This is necessary for "batch" Builders that can be called repeatedly 417 to build up a list of matching target and source files that will be 418 used in order to update multiple target files at once from multiple 419 corresponding source files, for tools like MSVC that support it.""" 420 self.batches.append(Batch(targets, sources))
421
422 - def prepare(self):
423 """ 424 Preparatory checks for whether this Executor can go ahead 425 and (try to) build its targets. 426 """ 427 for s in self.get_all_sources(): 428 if s.missing(): 429 msg = "Source `%s' not found, needed by target `%s'." 430 raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0]))
431
432 - def add_pre_action(self, action):
433 self.pre_actions.append(action)
434
435 - def add_post_action(self, action):
436 self.post_actions.append(action)
437 438 # another extra indirection for new-style objects and nullify... 439
440 - def __str__(self):
441 return _execute_str_map[self._execute_str](self)
442
443 - def nullify(self):
444 self.cleanup() 445 self._do_execute = 0 446 self._execute_str = 0
447 448 @SCons.Memoize.CountMethodCall
449 - def get_contents(self):
450 """Fetch the signature contents. This is the main reason this 451 class exists, so we can compute this once and cache it regardless 452 of how many target or source Nodes there are. 453 454 Returns bytes 455 """ 456 try: 457 return self._memo['get_contents'] 458 except KeyError: 459 pass 460 env = self.get_build_env() 461 462 action_list = self.get_action_list() 463 all_targets = self.get_all_targets() 464 all_sources = self.get_all_sources() 465 466 result = bytearray("",'utf-8').join([action.get_contents(all_targets, 467 all_sources, 468 env) 469 for action in action_list]) 470 471 self._memo['get_contents'] = result 472 return result
473
474 - def get_timestamp(self):
475 """Fetch a time stamp for this Executor. We don't have one, of 476 course (only files do), but this is the interface used by the 477 timestamp module. 478 """ 479 return 0
480
481 - def scan_targets(self, scanner):
482 # TODO(batch): scan by batches 483 self.scan(scanner, self.get_all_targets())
484
485 - def scan_sources(self, scanner):
486 # TODO(batch): scan by batches 487 if self.batches[0].sources: 488 self.scan(scanner, self.get_all_sources())
489
490 - def scan(self, scanner, node_list):
491 """Scan a list of this Executor's files (targets or sources) for 492 implicit dependencies and update all of the targets with them. 493 This essentially short-circuits an N*M scan of the sources for 494 each individual target, which is a hell of a lot more efficient. 495 """ 496 env = self.get_build_env() 497 path = self.get_build_scanner_path 498 kw = self.get_kw() 499 500 # TODO(batch): scan by batches) 501 deps = [] 502 503 for node in node_list: 504 node.disambiguate() 505 deps.extend(node.get_implicit_deps(env, scanner, path, kw)) 506 507 deps.extend(self.get_implicit_deps()) 508 509 for tgt in self.get_all_targets(): 510 tgt.add_to_implicit(deps)
511
512 - def _get_unignored_sources_key(self, node, ignore=()):
513 return (node,) + tuple(ignore)
514 515 @SCons.Memoize.CountDictCall(_get_unignored_sources_key)
516 - def get_unignored_sources(self, node, ignore=()):
517 key = (node,) + tuple(ignore) 518 try: 519 memo_dict = self._memo['get_unignored_sources'] 520 except KeyError: 521 memo_dict = {} 522 self._memo['get_unignored_sources'] = memo_dict 523 else: 524 try: 525 return memo_dict[key] 526 except KeyError: 527 pass 528 529 if node: 530 # TODO: better way to do this (it's a linear search, 531 # but it may not be critical path)? 532 sourcelist = [] 533 for b in self.batches: 534 if node in b.targets: 535 sourcelist = b.sources 536 break 537 else: 538 sourcelist = self.get_all_sources() 539 if ignore: 540 idict = {} 541 for i in ignore: 542 idict[i] = 1 543 sourcelist = [s for s in sourcelist if s not in idict] 544 545 memo_dict[key] = sourcelist 546 547 return sourcelist
548
549 - def get_implicit_deps(self):
550 """Return the executor's implicit dependencies, i.e. the nodes of 551 the commands to be executed.""" 552 result = [] 553 build_env = self.get_build_env() 554 for act in self.get_action_list(): 555 deps = act.get_implicit_deps(self.get_all_targets(), 556 self.get_all_sources(), 557 build_env) 558 result.extend(deps) 559 return result
560 561 562 563 _batch_executors = {}
564 565 -def GetBatchExecutor(key):
566 return _batch_executors[key]
567
568 -def AddBatchExecutor(key, executor):
569 assert key not in _batch_executors 570 _batch_executors[key] = executor
571 572 nullenv = None 573 574 575 import SCons.Util
576 -class NullEnvironment(SCons.Util.Null):
577 import SCons.CacheDir 578 _CacheDir_path = None 579 _CacheDir = SCons.CacheDir.CacheDir(None)
580 - def get_CacheDir(self):
581 return self._CacheDir
582
583 584 -def get_NullEnvironment():
585 """Use singleton pattern for Null Environments.""" 586 global nullenv 587 588 if nullenv is None: 589 nullenv = NullEnvironment() 590 return nullenv
591
592 -class Null(object, with_metaclass(NoSlotsPyPy)):
593 """A null Executor, with a null build Environment, that does 594 nothing when the rest of the methods call it. 595 596 This might be able to disappear when we refactor things to 597 disassociate Builders from Nodes entirely, so we're not 598 going to worry about unit tests for this--at least for now. 599 """ 600 601 __slots__ = ('pre_actions', 602 'post_actions', 603 'env', 604 'overridelist', 605 'batches', 606 'builder_kw', 607 '_memo', 608 'lvars', 609 '_changed_sources_list', 610 '_changed_targets_list', 611 '_unchanged_sources_list', 612 '_unchanged_targets_list', 613 'action_list', 614 '_do_execute', 615 '_execute_str') 616
617 - def __init__(self, *args, **kw):
618 if SCons.Debug.track_instances: logInstanceCreation(self, 'Executor.Null') 619 self.batches = [Batch(kw['targets'][:], [])]
620 - def get_build_env(self):
621 return get_NullEnvironment()
622 - def get_build_scanner_path(self):
623 return None
624 - def cleanup(self):
625 pass
626 - def prepare(self):
627 pass
628 - def get_unignored_sources(self, *args, **kw):
629 return tuple(())
630 - def get_action_targets(self):
631 return []
632 - def get_action_list(self):
633 return []
634 - def get_all_targets(self):
635 return self.batches[0].targets
636 - def get_all_sources(self):
637 return self.batches[0].targets[0].sources
638 - def get_all_children(self):
639 return self.batches[0].targets[0].children()
640 - def get_all_prerequisites(self):
641 return []
642 - def get_action_side_effects(self):
643 return []
644 - def __call__(self, *args, **kw):
645 return 0
646 - def get_contents(self):
647 return ''
648 - def _morph(self):
649 """Morph this Null executor to a real Executor object.""" 650 batches = self.batches 651 self.__class__ = Executor 652 self.__init__([]) 653 self.batches = batches
654 655 # The following methods require morphing this Null Executor to a 656 # real Executor object. 657
658 - def add_pre_action(self, action):
659 self._morph() 660 self.add_pre_action(action)
661 - def add_post_action(self, action):
662 self._morph() 663 self.add_post_action(action)
664 - def set_action_list(self, action):
665 self._morph() 666 self.set_action_list(action)
667 668 # Local Variables: 669 # tab-width:4 670 # indent-tabs-mode:nil 671 # End: 672 # vim: set expandtab tabstop=4 shiftwidth=4: 673