SCons.CacheDir

1 # 2 # Copyright (c) 2001 - 2019 The SCons Foundation 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining 5 # a copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, 8 # distribute, sublicense, and/or sell copies of the Software, and to 9 # permit persons to whom the Software is furnished to do so, subject to 10 # the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 # 23 24 __revision__ = "src/engine/SCons/CacheDir.py 3a41ed6b288cee8d085373ad7fa02894e1903864 2019-01-23 17:30:35 bdeegan" 25 26 __doc__ = """ 27 CacheDir support 28 """ 29 30 import hashlib 31 import json 32 import os 33 import stat 34 import sys 35 36 import SCons.Action 37 import SCons.Warnings 38 39 cache_enabled = True 40 cache_debug = False 41 cache_force = False 42 cache_show = False 43 cache_readonly = False 44

45 -def CacheRetrieveFunc(target, source, env):

46 t = target[0] 47 fs = t.fs 48 cd = env.get_CacheDir() 49 cachedir, cachefile = cd.cachepath(t) 50 if not fs.exists(cachefile): 51 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 52 return 1 53 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 54 if SCons.Action.execute_actions: 55 if fs.islink(cachefile): 56 fs.symlink(fs.readlink(cachefile), t.get_internal_path()) 57 else: 58 env.copy_from_cache(cachefile, t.get_internal_path()) 59 try: 60 os.utime(cachefile, None) 61 except OSError: 62 pass 63 st = fs.stat(cachefile) 64 fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 65 return 0

66

67 -def CacheRetrieveString(target, source, env):

68 t = target[0] 69 fs = t.fs 70 cd = env.get_CacheDir() 71 cachedir, cachefile = cd.cachepath(t) 72 if t.fs.exists(cachefile): 73 return "Retrieved `%s' from cache" % t.get_internal_path() 74 return None

75 76 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 77 78 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) 79

80 -def CachePushFunc(target, source, env):

81 if cache_readonly: 82 return 83 84 t = target[0] 85 if t.nocache: 86 return 87 fs = t.fs 88 cd = env.get_CacheDir() 89 cachedir, cachefile = cd.cachepath(t) 90 if fs.exists(cachefile): 91 # Don't bother copying it if it's already there. Note that 92 # usually this "shouldn't happen" because if the file already 93 # existed in cache, we'd have retrieved the file from there, 94 # not built it. This can happen, though, in a race, if some 95 # other person running the same build pushes their copy to 96 # the cache after we decide we need to build it but before our 97 # build completes. 98 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 99 return 100 101 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 102 103 tempfile = cachefile+'.tmp'+str(os.getpid()) 104 errfmt = "Unable to copy %s to cache. Cache file is %s" 105 106 if not fs.isdir(cachedir): 107 try: 108 fs.makedirs(cachedir) 109 except EnvironmentError: 110 # We may have received an exception because another process 111 # has beaten us creating the directory. 112 if not fs.isdir(cachedir): 113 msg = errfmt % (str(target), cachefile) 114 raise SCons.Errors.EnvironmentError(msg) 115 116 try: 117 if fs.islink(t.get_internal_path()): 118 fs.symlink(fs.readlink(t.get_internal_path()), tempfile) 119 else: 120 fs.copy2(t.get_internal_path(), tempfile) 121 fs.rename(tempfile, cachefile) 122 st = fs.stat(t.get_internal_path()) 123 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 124 except EnvironmentError: 125 # It's possible someone else tried writing the file at the 126 # same time we did, or else that there was some problem like 127 # the CacheDir being on a separate file system that's full. 128 # In any case, inability to push a file to cache doesn't affect 129 # the correctness of the build, so just print a warning. 130 msg = errfmt % (str(target), cachefile) 131 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)

132 133 CachePush = SCons.Action.Action(CachePushFunc, None) 134 135 # Nasty hack to cut down to one warning for each cachedir path that needs 136 # upgrading. 137 warned = dict() 138

139 -class CacheDir(object):

140

141 - def __init__(self, path):

142 self.path = path 143 self.current_cache_debug = None 144 self.debugFP = None 145 self.config = dict() 146 if path is None: 147 return 148 # See if there's a config file in the cache directory. If there is, 149 # use it. If there isn't, and the directory exists and isn't empty, 150 # produce a warning. If the directory doesn't exist or is empty, 151 # write a config file. 152 config_file = os.path.join(path, 'config') 153 if not os.path.exists(config_file): 154 # A note: There is a race hazard here, if two processes start and 155 # attempt to create the cache directory at the same time. However, 156 # python doesn't really give you the option to do exclusive file 157 # creation (it doesn't even give you the option to error on opening 158 # an existing file for writing...). The ordering of events here 159 # as an attempt to alleviate this, on the basis that it's a pretty 160 # unlikely occurence (it'd require two builds with a brand new cache 161 # directory) 162 if os.path.isdir(path) and len(os.listdir(path)) != 0: 163 self.config['prefix_len'] = 1 164 # When building the project I was testing this on, the warning 165 # was output over 20 times. That seems excessive 166 global warned 167 if self.path not in warned: 168 msg = "Please upgrade your cache by running " +\ 169 " scons-configure-cache.py " + self.path 170 SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) 171 warned[self.path] = True 172 else: 173 if not os.path.isdir(path): 174 try: 175 os.makedirs(path) 176 except OSError: 177 # If someone else is trying to create the directory at 178 # the same time as me, bad things will happen 179 msg = "Failed to create cache directory " + path 180 raise SCons.Errors.EnvironmentError(msg) 181 182 self.config['prefix_len'] = 2 183 if not os.path.exists(config_file): 184 try: 185 with open(config_file, 'w') as config: 186 json.dump(self.config, config) 187 except: 188 msg = "Failed to write cache configuration for " + path 189 raise SCons.Errors.EnvironmentError(msg) 190 else: 191 try: 192 with open(config_file) as config: 193 self.config = json.load(config) 194 except ValueError: 195 msg = "Failed to read cache configuration for " + path 196 raise SCons.Errors.EnvironmentError(msg)

197 198

199 - def CacheDebug(self, fmt, target, cachefile):

200 if cache_debug != self.current_cache_debug: 201 if cache_debug == '-': 202 self.debugFP = sys.stdout 203 elif cache_debug: 204 self.debugFP = open(cache_debug, 'w') 205 else: 206 self.debugFP = None 207 self.current_cache_debug = cache_debug 208 if self.debugFP: 209 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))

210

211 - def is_enabled(self):

212 return cache_enabled and not self.path is None

213

214 - def is_readonly(self):

215 return cache_readonly

216

217 - def cachepath(self, node):

218 """ 219 """ 220 if not self.is_enabled(): 221 return None, None 222 223 sig = node.get_cachedir_bsig() 224 225 subdir = sig[:self.config['prefix_len']].upper() 226 227 dir = os.path.join(self.path, subdir) 228 return dir, os.path.join(dir, sig)

229

230 - def retrieve(self, node):

231 """ 232 This method is called from multiple threads in a parallel build, 233 so only do thread safe stuff here. Do thread unsafe stuff in 234 built(). 235 236 Note that there's a special trick here with the execute flag 237 (one that's not normally done for other actions). Basically 238 if the user requested a no_exec (-n) build, then 239 SCons.Action.execute_actions is set to 0 and when any action 240 is called, it does its showing but then just returns zero 241 instead of actually calling the action execution operation. 242 The problem for caching is that if the file does NOT exist in 243 cache then the CacheRetrieveString won't return anything to 244 show for the task, but the Action.__call__ won't call 245 CacheRetrieveFunc; instead it just returns zero, which makes 246 the code below think that the file *was* successfully 247 retrieved from the cache, therefore it doesn't do any 248 subsequent building. However, the CacheRetrieveString didn't 249 print anything because it didn't actually exist in the cache, 250 and no more build actions will be performed, so the user just 251 sees nothing. The fix is to tell Action.__call__ to always 252 execute the CacheRetrieveFunc and then have the latter 253 explicitly check SCons.Action.execute_actions itself. 254 """ 255 if not self.is_enabled(): 256 return False 257 258 env = node.get_build_env() 259 if cache_show: 260 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 261 node.build(presub=0, execute=0) 262 return True 263 else: 264 if CacheRetrieve(node, [], env, execute=1) == 0: 265 return True 266 267 return False

268

269 - def push(self, node):

270 if self.is_readonly() or not self.is_enabled(): 271 return 272 return CachePush(node, [], node.get_build_env())

273

274 - def push_if_forced(self, node):

275 if cache_force: 276 return self.push(node)

277 278 # Local Variables: 279 # tab-width:4 280 # indent-tabs-mode:nil 281 # End: 282 # vim: set expandtab tabstop=4 shiftwidth=4: 283

Source Code for Module SCons.CacheDir