SCons.CacheDir

1 # 2 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 The SCons Foundation 3 # 4 # Permission is hereby granted, free of charge, to any person obtaining 5 # a copy of this software and associated documentation files (the 6 # "Software"), to deal in the Software without restriction, including 7 # without limitation the rights to use, copy, modify, merge, publish, 8 # distribute, sublicense, and/or sell copies of the Software, and to 9 # permit persons to whom the Software is furnished to do so, subject to 10 # the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included 13 # in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 20 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 21 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 # 23 24 __revision__ = "src/engine/SCons/CacheDir.py 5023 2010/06/14 22:05:46 scons" 25 26 __doc__ = """ 27 CacheDir support 28 """ 29 30 import os.path 31 import stat 32 import sys 33 34 import SCons.Action 35 36 cache_enabled = True 37 cache_debug = False 38 cache_force = False 39 cache_show = False 40

41 -def CacheRetrieveFunc(target, source, env):

42 t = target[0] 43 fs = t.fs 44 cd = env.get_CacheDir() 45 cachedir, cachefile = cd.cachepath(t) 46 if not fs.exists(cachefile): 47 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 48 return 1 49 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 50 if SCons.Action.execute_actions: 51 if fs.islink(cachefile): 52 fs.symlink(fs.readlink(cachefile), t.path) 53 else: 54 env.copy_from_cache(cachefile, t.path) 55 st = fs.stat(cachefile) 56 fs.chmod(t.path, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 57 return 0

58

59 -def CacheRetrieveString(target, source, env):

60 t = target[0] 61 fs = t.fs 62 cd = env.get_CacheDir() 63 cachedir, cachefile = cd.cachepath(t) 64 if t.fs.exists(cachefile): 65 return "Retrieved `%s' from cache" % t.path 66 return None

67 68 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 69 70 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) 71

72 -def CachePushFunc(target, source, env):

73 t = target[0] 74 if t.nocache: 75 return 76 fs = t.fs 77 cd = env.get_CacheDir() 78 cachedir, cachefile = cd.cachepath(t) 79 if fs.exists(cachefile): 80 # Don't bother copying it if it's already there. Note that 81 # usually this "shouldn't happen" because if the file already 82 # existed in cache, we'd have retrieved the file from there, 83 # not built it. This can happen, though, in a race, if some 84 # other person running the same build pushes their copy to 85 # the cache after we decide we need to build it but before our 86 # build completes. 87 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 88 return 89 90 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 91 92 tempfile = cachefile+'.tmp'+str(os.getpid()) 93 errfmt = "Unable to copy %s to cache. Cache file is %s" 94 95 if not fs.isdir(cachedir): 96 try: 97 fs.makedirs(cachedir) 98 except EnvironmentError: 99 # We may have received an exception because another process 100 # has beaten us creating the directory. 101 if not fs.isdir(cachedir): 102 msg = errfmt % (str(target), cachefile) 103 raise SCons.Errors.EnvironmentError(msg) 104 105 try: 106 if fs.islink(t.path): 107 fs.symlink(fs.readlink(t.path), tempfile) 108 else: 109 fs.copy2(t.path, tempfile) 110 fs.rename(tempfile, cachefile) 111 st = fs.stat(t.path) 112 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 113 except EnvironmentError: 114 # It's possible someone else tried writing the file at the 115 # same time we did, or else that there was some problem like 116 # the CacheDir being on a separate file system that's full. 117 # In any case, inability to push a file to cache doesn't affect 118 # the correctness of the build, so just print a warning. 119 msg = errfmt % (str(target), cachefile) 120 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)

121 122 CachePush = SCons.Action.Action(CachePushFunc, None) 123

124 -class CacheDir(object):

125

126 - def __init__(self, path):

127 try: 128 import hashlib 129 except ImportError: 130 msg = "No hashlib or MD5 module available, CacheDir() not supported" 131 SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) 132 self.path = None 133 else: 134 self.path = path 135 self.current_cache_debug = None 136 self.debugFP = None

137

138 - def CacheDebug(self, fmt, target, cachefile):

139 if cache_debug != self.current_cache_debug: 140 if cache_debug == '-': 141 self.debugFP = sys.stdout 142 elif cache_debug: 143 self.debugFP = open(cache_debug, 'w') 144 else: 145 self.debugFP = None 146 self.current_cache_debug = cache_debug 147 if self.debugFP: 148 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))

149

150 - def is_enabled(self):

151 return (cache_enabled and not self.path is None)

152

153 - def cachepath(self, node):

154 """ 155 """ 156 if not self.is_enabled(): 157 return None, None 158 159 sig = node.get_cachedir_bsig() 160 subdir = sig[0].upper() 161 dir = os.path.join(self.path, subdir) 162 return dir, os.path.join(dir, sig)

163

164 - def retrieve(self, node):

165 """ 166 This method is called from multiple threads in a parallel build, 167 so only do thread safe stuff here. Do thread unsafe stuff in 168 built(). 169 170 Note that there's a special trick here with the execute flag 171 (one that's not normally done for other actions). Basically 172 if the user requested a no_exec (-n) build, then 173 SCons.Action.execute_actions is set to 0 and when any action 174 is called, it does its showing but then just returns zero 175 instead of actually calling the action execution operation. 176 The problem for caching is that if the file does NOT exist in 177 cache then the CacheRetrieveString won't return anything to 178 show for the task, but the Action.__call__ won't call 179 CacheRetrieveFunc; instead it just returns zero, which makes 180 the code below think that the file *was* successfully 181 retrieved from the cache, therefore it doesn't do any 182 subsequent building. However, the CacheRetrieveString didn't 183 print anything because it didn't actually exist in the cache, 184 and no more build actions will be performed, so the user just 185 sees nothing. The fix is to tell Action.__call__ to always 186 execute the CacheRetrieveFunc and then have the latter 187 explicitly check SCons.Action.execute_actions itself. 188 """ 189 if not self.is_enabled(): 190 return False 191 192 env = node.get_build_env() 193 if cache_show: 194 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 195 node.build(presub=0, execute=0) 196 return True 197 else: 198 if CacheRetrieve(node, [], env, execute=1) == 0: 199 return True 200 201 return False

202

203 - def push(self, node):

204 if not self.is_enabled(): 205 return 206 return CachePush(node, [], node.get_build_env())

207

208 - def push_if_forced(self, node):

209 if cache_force: 210 return self.push(node)

211 212 # Local Variables: 213 # tab-width:4 214 # indent-tabs-mode:nil 215 # End: 216 # vim: set expandtab tabstop=4 shiftwidth=4: 217

Source Code for Module SCons.CacheDir