1
# (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
import stat, os, sha, time
19
from trace import mutter
20
from errors import BzrError, BzrCheckError
23
"""File stat cache to speed up tree comparisons.
25
This module basically gives a quick way to find the SHA-1 and related
26
information of a file in the working directory, without actually
27
reading and hashing the whole file.
34
Users of this module should not need to know about how this is
35
implemented, and in particular should not depend on the particular
36
data which is stored or its format.
38
This is done by maintaining a cache indexed by a file fingerprint of
39
(path, size, mtime, ctime, ino, dev) pointing to the SHA-1. If the
40
fingerprint has changed, we assume the file content has not changed
41
either and the SHA-1 is therefore the same.
43
If any of the fingerprint fields have changed then the file content
44
*may* have changed, or it may not have. We need to reread the file
45
contents to make sure, but this is not visible to the user or
46
higher-level code (except as a delay of course).
48
The mtime and ctime are stored with nanosecond fields, but not all
49
filesystems give this level of precision. There is therefore a
50
possible race: the file might be modified twice within a second
51
without changing the size or mtime, and a SHA-1 cached from the first
52
version would be wrong. We handle this by not recording a cached hash
53
for any files which were modified in the current second and that
54
therefore have the chance to change again before the second is up.
56
The only known hole in this design is if the system clock jumps
57
backwards crossing invocations of bzr. Please don't do that; use ntp
58
to gradually adjust your clock or don't use bzr over the step.
60
At the moment this is stored in a simple textfile; it might be nice
63
The cache is represented as a map from file_id to a tuple of (file_id,
64
sha1, path, size, mtime, ctime, ino, dev).
66
The SHA-1 is stored in memory as a hexdigest.
68
File names and file-ids are written out with non-ascii or whitespace
69
characters given as python-style unicode escapes. (file-ids shouldn't
70
contain wierd characters, but it might happen.)
73
# order of fields returned by fingerprint()
80
# order of fields in the statcache file and in the in-memory map
92
CACHE_HEADER = "### bzr statcache v3"
95
def fingerprint(abspath):
97
fs = os.lstat(abspath)
99
# might be missing, etc
102
if stat.S_ISDIR(fs.st_mode):
105
return (fs.st_size, fs.st_mtime,
106
fs.st_ctime, fs.st_ino, fs.st_dev)
111
return s.encode('unicode_escape') \
112
.replace('\n', '\\u000a') \
113
.replace(' ', '\\u0020') \
114
.replace('\r', '\\u000d')
117
def _write_cache(basedir, entry_iter, dangerfiles):
118
from atomicfile import AtomicFile
120
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
121
outf = AtomicFile(cachefn, 'wb')
122
outf.write(CACHE_HEADER + '\n')
124
for entry in entry_iter:
126
raise ValueError("invalid statcache entry tuple %r" % entry)
128
if entry[SC_FILE_ID] in dangerfiles:
129
continue # changed too recently
130
outf.write(safe_quote(entry[0])) # file id
132
outf.write(entry[1]) # hex sha1
134
outf.write(safe_quote(entry[2])) # name
136
outf.write(' %d' % nf)
145
def _write_cache_maybe(basedir, entry_iter, dangerfiles):
147
return _write_cache(basedir, entry_iter, dangerfiles)
149
mutter("cannot update statcache in %s: %s" % (basedir, e))
151
mutter("cannot update statcache in %s: %s" % (basedir, e))
155
def load_cache(basedir):
160
sha_re = re.compile(r'[a-f0-9]{40}')
163
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
164
cachefile = open(cachefn, 'rb')
168
line1 = cachefile.readline().rstrip('\r\n')
169
if line1 != CACHE_HEADER:
170
mutter('cache header marker not found at top of %s' % cachefn)
176
file_id = f[0].decode('unicode_escape')
178
raise BzrCheckError("duplicated file_id in cache: {%s}" % file_id)
181
if len(text_sha) != 40 or not sha_re.match(text_sha):
182
raise BzrCheckError("invalid file SHA-1 in cache: %r" % text_sha)
184
path = f[2].decode('unicode_escape')
185
if path in seen_paths:
186
raise BzrCheckError("duplicated path in cache: %r" % path)
187
seen_paths[path] = True
189
entry = (file_id, text_sha, path) + tuple([long(x) for x in f[3:]])
191
raise ValueError("invalid statcache entry tuple %r" % entry)
193
cache[file_id] = entry
199
def _files_from_inventory(inv):
200
for path, ie in inv.iter_entries():
201
if ie.kind != 'file':
203
yield ie.file_id, path
207
def update_cache(basedir, inv, flush=False):
208
"""Update and return the cache for the branch.
210
The returned cache may contain entries that have not been written
211
to disk for files recently touched.
213
flush -- discard any previous cache and recalculate from scratch.
217
# TODO: It's supposed to be faster to stat the files in order by inum.
218
# We don't directly know the inum of the files of course but we do
219
# know where they were last sighted, so we can sort by that.
221
assert isinstance(flush, bool)
225
cache = load_cache(basedir)
226
return _update_cache_from_list(basedir, cache, _files_from_inventory(inv))
230
def _update_cache_from_list(basedir, cache, to_update):
231
"""Update and return the cache for given files.
233
cache -- Previously cached values to be validated.
235
to_update -- Sequence of (file_id, path) pairs to check.
237
stat_cnt = missing_cnt = hardcheck = change_cnt = 0
239
# dangerfiles have been recently touched and can't be committed to
240
# a persistent cache yet, but they are returned to the caller.
243
now = int(time.time())
245
## mutter('update statcache under %r' % basedir)
246
for file_id, path in to_update:
247
abspath = os.path.join(basedir, path)
248
fp = fingerprint(abspath)
251
cacheentry = cache.get(file_id)
253
if fp == None: # not here
260
if (fp[FP_MTIME] >= now) or (fp[FP_CTIME] >= now):
261
dangerfiles[file_id] = True
263
if cacheentry and (cacheentry[3:] == fp):
264
continue # all stat fields unchanged
268
dig = sha.new(file(abspath, 'rb').read()).hexdigest()
270
# We update the cache even if the digest has not changed from
271
# last time we looked, so that the fingerprint fields will
273
cacheentry = (file_id, dig, path) + fp
274
cache[file_id] = cacheentry
277
mutter('statcache: statted %d files, read %d files, %d changed, %d dangerous, '
279
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles), len(cache)))
282
mutter('updating on-disk statcache')
283
_write_cache_maybe(basedir, cache.itervalues(), dangerfiles)
added
removed
bzrlib/statcache.py
