1
# (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
import stat, os, sha, time
19
from trace import mutter
20
from errors import BzrError, BzrCheckError
23
"""File stat cache to speed up tree comparisons.
25
This module basically gives a quick way to find the SHA-1 and related
26
information of a file in the working directory, without actually
27
reading and hashing the whole file. The information is validated by
28
checking the size, mtime, ctime, etc of the file as returned by the
31
This has no relation to the deprecated standard Python module called
32
statcache (vs bzrlib.statcache).
39
Users of this module should not need to know about how this is
40
implemented, and in particular should not depend on the particular
41
data which is stored or its format.
43
The cache maintains a mapping from filename to the SHA-1 of the
46
The cache also stores a fingerprint of (size, mtime, ctime, ino, dev)
47
which is used to validate that the entry is up-to-date.
51
This is done by maintaining a cache indexed by a file fingerprint of
52
(path, size, mtime, ctime, ino, dev) pointing to the SHA-1. If the
53
fingerprint has changed, we assume the file content has not changed
54
either and the SHA-1 is therefore the same.
56
If any of the fingerprint fields have changed then the file content
57
*may* have changed, or it may not have. We need to reread the file
58
contents to make sure, but this is not visible to the user or
59
higher-level code (except as a delay of course).
61
The mtime and ctime are stored with nanosecond fields, but not all
62
filesystems give this level of precision. There is therefore a
63
possible race: the file might be modified twice within a second
64
without changing the size or mtime, and a SHA-1 cached from the first
65
version would be wrong. We handle this by not recording a cached hash
66
for any files which were modified in the current second and that
67
therefore have the chance to change again before the second is up.
69
The only known hole in this design is if the system clock jumps
70
backwards crossing invocations of bzr. Please don't do that; use ntp
71
to gradually adjust your clock or don't use bzr over the step.
73
At the moment this is stored in a simple textfile; it might be nice
74
to use a tdb instead to allow faster lookup by file-id.
76
The cache is represented as a map from file_id to a tuple of (file_id,
77
sha1, path, size, mtime, ctime, ino, dev).
79
The SHA-1 is stored in memory as a hexdigest.
81
This version of the file on disk has one line per record, and fields
82
separated by \0 records.
85
# order of fields returned by fingerprint()
92
# order of fields in the statcache file and in the in-memory map
104
CACHE_HEADER = "### bzr statcache v4"
107
def fingerprint(abspath):
109
fs = os.lstat(abspath)
111
# might be missing, etc
114
if stat.S_ISDIR(fs.st_mode):
117
return (fs.st_size, fs.st_mtime,
118
fs.st_ctime, fs.st_ino, fs.st_dev)
122
def _write_cache(basedir, entries):
123
from atomicfile import AtomicFile
125
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
126
outf = AtomicFile(cachefn, 'wb')
128
outf.write(CACHE_HEADER + '\n')
130
for entry in entries:
132
raise ValueError("invalid statcache entry tuple %r" % entry)
133
outf.write(entry[0].encode('utf-8')) # file id
135
outf.write(entry[1]) # hex sha1
137
outf.write(entry[2].encode('utf-8')) # name
139
outf.write('\0%d' % nf)
148
def _try_write_cache(basedir, entries):
150
return _write_cache(basedir, entries)
152
mutter("cannot update statcache in %s: %s" % (basedir, e))
154
mutter("cannot update statcache in %s: %s" % (basedir, e))
158
def load_cache(basedir):
162
from bzrlib.trace import warning
164
assert isinstance(basedir, basestring)
166
sha_re = re.compile(r'[a-f0-9]{40}')
169
cachefn = os.path.join(basedir, '.bzr', 'stat-cache')
170
cachefile = open(cachefn, 'rb')
174
line1 = cachefile.readline().rstrip('\r\n')
175
if line1 != CACHE_HEADER:
176
mutter('cache header marker not found at top of %s; discarding cache'
183
file_id = f[0].decode('utf-8')
185
warning("duplicated file_id in cache: {%s}" % file_id)
188
if len(text_sha) != 40 or not sha_re.match(text_sha):
189
raise BzrCheckError("invalid file SHA-1 in cache: %r" % text_sha)
191
path = f[2].decode('utf-8')
192
if path in seen_paths:
193
warning("duplicated path in cache: %r" % path)
194
seen_paths[path] = True
196
entry = (file_id, text_sha, path) + tuple([long(x) for x in f[3:]])
198
raise ValueError("invalid statcache entry tuple %r" % entry)
200
cache[file_id] = entry
205
def _files_from_inventory(inv):
206
for path, ie in inv.iter_entries():
207
if ie.kind != 'file':
209
yield ie.file_id, path
213
def update_cache(basedir, inv, flush=False):
214
"""Update and return the cache for the branch.
216
The returned cache may contain entries that have not been written
217
to disk for files recently touched.
219
flush -- discard any previous cache and recalculate from scratch.
222
# load the existing cache; use information there to find a list of
223
# files ordered by inode, which is alleged to be the fastest order
226
to_update = _files_from_inventory(inv)
228
assert isinstance(flush, bool)
232
cache = load_cache(basedir)
236
for file_id, path in to_update:
238
by_inode.append((cache[file_id][SC_INO], file_id, path))
240
without_inode.append((file_id, path))
243
to_update = [a[1:] for a in by_inode] + without_inode
245
stat_cnt = missing_cnt = new_cnt = hardcheck = change_cnt = 0
247
# dangerfiles have been recently touched and can't be committed to
248
# a persistent cache yet, but they are returned to the caller.
251
now = int(time.time())
253
## mutter('update statcache under %r' % basedir)
254
for file_id, path in to_update:
255
abspath = os.path.join(basedir, path)
256
fp = fingerprint(abspath)
259
cacheentry = cache.get(file_id)
261
if fp == None: # not here
270
if (fp[FP_MTIME] >= now) or (fp[FP_CTIME] >= now):
271
dangerfiles.append(file_id)
273
if cacheentry and (cacheentry[3:] == fp):
274
continue # all stat fields unchanged
278
dig = sha.new(file(abspath, 'rb').read()).hexdigest()
280
# We update the cache even if the digest has not changed from
281
# last time we looked, so that the fingerprint fields will
283
cacheentry = (file_id, dig, path) + fp
284
cache[file_id] = cacheentry
287
mutter('statcache: statted %d files, read %d files, %d changed, %d dangerous, '
288
'%d deleted, %d new, '
290
% (stat_cnt, hardcheck, change_cnt, len(dangerfiles),
291
missing_cnt, new_cnt, len(cache)))
294
mutter('updating on-disk statcache')
297
safe_cache = cache.copy()
298
for file_id in dangerfiles:
299
del safe_cache[file_id]
303
_try_write_cache(basedir, safe_cache.itervalues())
added
removed
bzrlib/statcache.py
