SCons.CacheDir

1 # 2 # Copyright (c) 2001 - 2017 The SCons Foundation 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining 5 # a copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, 8 # distribute, sublicense, and/or sell copies of the Software, and to 9 # permit persons to whom the Software is furnished to do so, subject to 10 # the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 # 23 24 __revision__ = "src/engine/SCons/CacheDir.py 74b2c53bc42290e911b334a6b44f187da698a668 2017/11/14 13:16:53 bdbaddog" 25 26 __doc__ = """ 27 CacheDir support 28 """ 29 30 import json 31 import os 32 import stat 33 import sys 34 35 import SCons.Action 36 import SCons.Warnings 37 38 cache_enabled = True 39 cache_debug = False 40 cache_force = False 41 cache_show = False 42 cache_readonly = False 43

44 -def CacheRetrieveFunc(target, source, env):

45 t = target[0] 46 fs = t.fs 47 cd = env.get_CacheDir() 48 cachedir, cachefile = cd.cachepath(t) 49 if not fs.exists(cachefile): 50 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 51 return 1 52 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 53 if SCons.Action.execute_actions: 54 if fs.islink(cachefile): 55 fs.symlink(fs.readlink(cachefile), t.get_internal_path()) 56 else: 57 env.copy_from_cache(cachefile, t.get_internal_path()) 58 st = fs.stat(cachefile) 59 fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 60 return 0

61

62 -def CacheRetrieveString(target, source, env):

63 t = target[0] 64 fs = t.fs 65 cd = env.get_CacheDir() 66 cachedir, cachefile = cd.cachepath(t) 67 if t.fs.exists(cachefile): 68 return "Retrieved `%s' from cache" % t.get_internal_path() 69 return None

70 71 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 72 73 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) 74

75 -def CachePushFunc(target, source, env):

76 if cache_readonly: 77 return 78 79 t = target[0] 80 if t.nocache: 81 return 82 fs = t.fs 83 cd = env.get_CacheDir() 84 cachedir, cachefile = cd.cachepath(t) 85 if fs.exists(cachefile): 86 # Don't bother copying it if it's already there. Note that 87 # usually this "shouldn't happen" because if the file already 88 # existed in cache, we'd have retrieved the file from there, 89 # not built it. This can happen, though, in a race, if some 90 # other person running the same build pushes their copy to 91 # the cache after we decide we need to build it but before our 92 # build completes. 93 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 94 return 95 96 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 97 98 tempfile = cachefile+'.tmp'+str(os.getpid()) 99 errfmt = "Unable to copy %s to cache. Cache file is %s" 100 101 if not fs.isdir(cachedir): 102 try: 103 fs.makedirs(cachedir) 104 except EnvironmentError: 105 # We may have received an exception because another process 106 # has beaten us creating the directory. 107 if not fs.isdir(cachedir): 108 msg = errfmt % (str(target), cachefile) 109 raise SCons.Errors.EnvironmentError(msg) 110 111 try: 112 if fs.islink(t.get_internal_path()): 113 fs.symlink(fs.readlink(t.get_internal_path()), tempfile) 114 else: 115 fs.copy2(t.get_internal_path(), tempfile) 116 fs.rename(tempfile, cachefile) 117 st = fs.stat(t.get_internal_path()) 118 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 119 except EnvironmentError: 120 # It's possible someone else tried writing the file at the 121 # same time we did, or else that there was some problem like 122 # the CacheDir being on a separate file system that's full. 123 # In any case, inability to push a file to cache doesn't affect 124 # the correctness of the build, so just print a warning. 125 msg = errfmt % (str(target), cachefile) 126 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)

127 128 CachePush = SCons.Action.Action(CachePushFunc, None) 129 130 # Nasty hack to cut down to one warning for each cachedir path that needs 131 # upgrading. 132 warned = dict() 133

134 -class CacheDir(object):

135

136 - def __init__(self, path):

137 try: 138 import hashlib 139 except ImportError: 140 msg = "No hashlib or MD5 module available, CacheDir() not supported" 141 SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) 142 path = None 143 self.path = path 144 self.current_cache_debug = None 145 self.debugFP = None 146 self.config = dict() 147 if path is None: 148 return 149 # See if there's a config file in the cache directory. If there is, 150 # use it. If there isn't, and the directory exists and isn't empty, 151 # produce a warning. If the directory doesn't exist or is empty, 152 # write a config file. 153 config_file = os.path.join(path, 'config') 154 if not os.path.exists(config_file): 155 # A note: There is a race hazard here, if two processes start and 156 # attempt to create the cache directory at the same time. However, 157 # python doesn't really give you the option to do exclusive file 158 # creation (it doesn't even give you the option to error on opening 159 # an existing file for writing...). The ordering of events here 160 # as an attempt to alleviate this, on the basis that it's a pretty 161 # unlikely occurence (it'd require two builds with a brand new cache 162 # directory) 163 if os.path.isdir(path) and len(os.listdir(path)) != 0: 164 self.config['prefix_len'] = 1 165 # When building the project I was testing this on, the warning 166 # was output over 20 times. That seems excessive 167 global warned 168 if self.path not in warned: 169 msg = "Please upgrade your cache by running " +\ 170 " scons-configure-cache.py " + self.path 171 SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) 172 warned[self.path] = True 173 else: 174 if not os.path.isdir(path): 175 try: 176 os.makedirs(path) 177 except OSError: 178 # If someone else is trying to create the directory at 179 # the same time as me, bad things will happen 180 msg = "Failed to create cache directory " + path 181 raise SCons.Errors.EnvironmentError(msg) 182 183 self.config['prefix_len'] = 2 184 if not os.path.exists(config_file): 185 try: 186 with open(config_file, 'w') as config: 187 json.dump(self.config, config) 188 except: 189 msg = "Failed to write cache configuration for " + path 190 raise SCons.Errors.EnvironmentError(msg) 191 else: 192 try: 193 with open(config_file) as config: 194 self.config = json.load(config) 195 except ValueError: 196 msg = "Failed to read cache configuration for " + path 197 raise SCons.Errors.EnvironmentError(msg)

198 199

200 - def CacheDebug(self, fmt, target, cachefile):

201 if cache_debug != self.current_cache_debug: 202 if cache_debug == '-': 203 self.debugFP = sys.stdout 204 elif cache_debug: 205 self.debugFP = open(cache_debug, 'w') 206 else: 207 self.debugFP = None 208 self.current_cache_debug = cache_debug 209 if self.debugFP: 210 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))

211

212 - def is_enabled(self):

213 return cache_enabled and not self.path is None

214

215 - def is_readonly(self):

216 return cache_readonly

217

218 - def cachepath(self, node):

219 """ 220 """ 221 if not self.is_enabled(): 222 return None, None 223 224 sig = node.get_cachedir_bsig() 225 subdir = sig[:self.config['prefix_len']].upper() 226 dir = os.path.join(self.path, subdir) 227 return dir, os.path.join(dir, sig)

228

229 - def retrieve(self, node):

230 """ 231 This method is called from multiple threads in a parallel build, 232 so only do thread safe stuff here. Do thread unsafe stuff in 233 built(). 234 235 Note that there's a special trick here with the execute flag 236 (one that's not normally done for other actions). Basically 237 if the user requested a no_exec (-n) build, then 238 SCons.Action.execute_actions is set to 0 and when any action 239 is called, it does its showing but then just returns zero 240 instead of actually calling the action execution operation. 241 The problem for caching is that if the file does NOT exist in 242 cache then the CacheRetrieveString won't return anything to 243 show for the task, but the Action.__call__ won't call 244 CacheRetrieveFunc; instead it just returns zero, which makes 245 the code below think that the file *was* successfully 246 retrieved from the cache, therefore it doesn't do any 247 subsequent building. However, the CacheRetrieveString didn't 248 print anything because it didn't actually exist in the cache, 249 and no more build actions will be performed, so the user just 250 sees nothing. The fix is to tell Action.__call__ to always 251 execute the CacheRetrieveFunc and then have the latter 252 explicitly check SCons.Action.execute_actions itself. 253 """ 254 if not self.is_enabled(): 255 return False 256 257 env = node.get_build_env() 258 if cache_show: 259 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 260 node.build(presub=0, execute=0) 261 return True 262 else: 263 if CacheRetrieve(node, [], env, execute=1) == 0: 264 return True 265 266 return False

267

268 - def push(self, node):

269 if self.is_readonly() or not self.is_enabled(): 270 return 271 return CachePush(node, [], node.get_build_env())

272

273 - def push_if_forced(self, node):

274 if cache_force: 275 return self.push(node)

276 277 # Local Variables: 278 # tab-width:4 279 # indent-tabs-mode:nil 280 # End: 281 # vim: set expandtab tabstop=4 shiftwidth=4: 282

Source Code for Module SCons.CacheDir