SCons.CacheDir

1 # 2 # Copyright (c) 2001 - 2015 The SCons Foundation 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining 5 # a copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, 8 # distribute, sublicense, and/or sell copies of the Software, and to 9 # permit persons to whom the Software is furnished to do so, subject to 10 # the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 # 23 24 __revision__ = "src/engine/SCons/CacheDir.py rel_2.4.0:3365:9259ea1c13d7 2015/09/21 14:03:43 bdbaddog" 25 26 __doc__ = """ 27 CacheDir support 28 """ 29 30 import os.path 31 import stat 32 import sys 33 34 import SCons.Action 35 36 cache_enabled = True 37 cache_debug = False 38 cache_force = False 39 cache_show = False 40 cache_readonly = False 41

42 -def CacheRetrieveFunc(target, source, env):

43 t = target[0] 44 fs = t.fs 45 cd = env.get_CacheDir() 46 cachedir, cachefile = cd.cachepath(t) 47 if not fs.exists(cachefile): 48 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 49 return 1 50 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 51 if SCons.Action.execute_actions: 52 if fs.islink(cachefile): 53 fs.symlink(fs.readlink(cachefile), t.get_internal_path()) 54 else: 55 env.copy_from_cache(cachefile, t.get_internal_path()) 56 st = fs.stat(cachefile) 57 fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 58 return 0

59

60 -def CacheRetrieveString(target, source, env):

61 t = target[0] 62 fs = t.fs 63 cd = env.get_CacheDir() 64 cachedir, cachefile = cd.cachepath(t) 65 if t.fs.exists(cachefile): 66 return "Retrieved `%s' from cache" % t.get_internal_path() 67 return None

68 69 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 70 71 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) 72

73 -def CachePushFunc(target, source, env):

74 if cache_readonly: return 75 76 t = target[0] 77 if t.nocache: 78 return 79 fs = t.fs 80 cd = env.get_CacheDir() 81 cachedir, cachefile = cd.cachepath(t) 82 if fs.exists(cachefile): 83 # Don't bother copying it if it's already there. Note that 84 # usually this "shouldn't happen" because if the file already 85 # existed in cache, we'd have retrieved the file from there, 86 # not built it. This can happen, though, in a race, if some 87 # other person running the same build pushes their copy to 88 # the cache after we decide we need to build it but before our 89 # build completes. 90 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 91 return 92 93 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 94 95 tempfile = cachefile+'.tmp'+str(os.getpid()) 96 errfmt = "Unable to copy %s to cache. Cache file is %s" 97 98 if not fs.isdir(cachedir): 99 try: 100 fs.makedirs(cachedir) 101 except EnvironmentError: 102 # We may have received an exception because another process 103 # has beaten us creating the directory. 104 if not fs.isdir(cachedir): 105 msg = errfmt % (str(target), cachefile) 106 raise SCons.Errors.EnvironmentError(msg) 107 108 try: 109 if fs.islink(t.get_internal_path()): 110 fs.symlink(fs.readlink(t.get_internal_path()), tempfile) 111 else: 112 fs.copy2(t.get_internal_path(), tempfile) 113 fs.rename(tempfile, cachefile) 114 st = fs.stat(t.get_internal_path()) 115 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 116 except EnvironmentError: 117 # It's possible someone else tried writing the file at the 118 # same time we did, or else that there was some problem like 119 # the CacheDir being on a separate file system that's full. 120 # In any case, inability to push a file to cache doesn't affect 121 # the correctness of the build, so just print a warning. 122 msg = errfmt % (str(target), cachefile) 123 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)

124 125 CachePush = SCons.Action.Action(CachePushFunc, None) 126

127 -class CacheDir(object):

128

129 - def __init__(self, path):

130 try: 131 import hashlib 132 except ImportError: 133 msg = "No hashlib or MD5 module available, CacheDir() not supported" 134 SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) 135 self.path = None 136 else: 137 self.path = path 138 self.current_cache_debug = None 139 self.debugFP = None

140

141 - def CacheDebug(self, fmt, target, cachefile):

142 if cache_debug != self.current_cache_debug: 143 if cache_debug == '-': 144 self.debugFP = sys.stdout 145 elif cache_debug: 146 self.debugFP = open(cache_debug, 'w') 147 else: 148 self.debugFP = None 149 self.current_cache_debug = cache_debug 150 if self.debugFP: 151 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))

152

153 - def is_enabled(self):

154 return (cache_enabled and not self.path is None)

155

156 - def is_readonly(self):

157 return cache_readonly

158

159 - def cachepath(self, node):

160 """ 161 """ 162 if not self.is_enabled(): 163 return None, None 164 165 sig = node.get_cachedir_bsig() 166 subdir = sig[0].upper() 167 dir = os.path.join(self.path, subdir) 168 return dir, os.path.join(dir, sig)

169

170 - def retrieve(self, node):

171 """ 172 This method is called from multiple threads in a parallel build, 173 so only do thread safe stuff here. Do thread unsafe stuff in 174 built(). 175 176 Note that there's a special trick here with the execute flag 177 (one that's not normally done for other actions). Basically 178 if the user requested a no_exec (-n) build, then 179 SCons.Action.execute_actions is set to 0 and when any action 180 is called, it does its showing but then just returns zero 181 instead of actually calling the action execution operation. 182 The problem for caching is that if the file does NOT exist in 183 cache then the CacheRetrieveString won't return anything to 184 show for the task, but the Action.__call__ won't call 185 CacheRetrieveFunc; instead it just returns zero, which makes 186 the code below think that the file *was* successfully 187 retrieved from the cache, therefore it doesn't do any 188 subsequent building. However, the CacheRetrieveString didn't 189 print anything because it didn't actually exist in the cache, 190 and no more build actions will be performed, so the user just 191 sees nothing. The fix is to tell Action.__call__ to always 192 execute the CacheRetrieveFunc and then have the latter 193 explicitly check SCons.Action.execute_actions itself. 194 """ 195 if not self.is_enabled(): 196 return False 197 198 env = node.get_build_env() 199 if cache_show: 200 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 201 node.build(presub=0, execute=0) 202 return True 203 else: 204 if CacheRetrieve(node, [], env, execute=1) == 0: 205 return True 206 207 return False

208

209 - def push(self, node):

210 if self.is_readonly() or not self.is_enabled(): 211 return 212 return CachePush(node, [], node.get_build_env())

213

214 - def push_if_forced(self, node):

215 if cache_force: 216 return self.push(node)

217 218 # Local Variables: 219 # tab-width:4 220 # indent-tabs-mode:nil 221 # End: 222 # vim: set expandtab tabstop=4 shiftwidth=4: 223

Source Code for Module SCons.CacheDir