1
# Copyright (C) 2005 by Canonical Development Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
Stores are the main data-storage mechanism for Bazaar-NG.
20
A store is a simple write-once container indexed by a universally
29
from stat import ST_SIZE
30
from StringIO import StringIO
32
from bzrlib.errors import BzrError
33
from bzrlib.trace import mutter
35
import bzrlib.osutils as osutils
36
from bzrlib.remotebranch import get_url
40
######################################################################
43
class StoreError(Exception):
48
"""An abstract store that holds files indexed by unique names.
50
Files can be added, but not modified once they are in. Typically
51
the hash is used as the name, or something else known to be unique,
54
>>> st = ImmutableScratchStore()
56
>>> st.add(StringIO('hello'), 'aa')
62
You are not allowed to add an id that is already present.
64
Entries can be retrieved as files, which may then be read.
66
>>> st.add(StringIO('goodbye'), '123123')
67
>>> st['123123'].read()
72
"""Return (count, bytes)
74
This is the (compressed) size stored on disk, not the size of
80
total += self._item_size(fid)
84
class ImmutableStore(Store):
85
"""Store that stores files on disk.
87
TODO: Atomic add by writing to a temporary file and renaming.
88
TODO: Guard against the same thing being stored twice, compressed and
89
uncompressed during copy_multi_immutable - the window is for a
90
matching store with some crack code that lets it offer a
91
non gz FOO and then a fz FOO.
93
In bzr 0.0.5 and earlier, files within the store were marked
94
readonly on disk. This is no longer done but existing stores need
98
def __init__(self, basedir):
99
super(ImmutableStore, self).__init__()
100
self._basedir = basedir
102
def _path(self, entry_id):
103
if not isinstance(entry_id, basestring):
104
raise TypeError(type(entry_id))
105
if '\\' in entry_id or '/' in entry_id:
106
raise ValueError("invalid store id %r" % entry_id)
107
return os.path.join(self._basedir, entry_id)
110
return "%s(%r)" % (self.__class__.__name__, self._basedir)
112
def add(self, f, fileid, compressed=True):
113
"""Add contents of a file into the store.
115
f -- An open file, or file-like object."""
116
# FIXME: Only works on files that will fit in memory
118
from bzrlib.atomicfile import AtomicFile
120
mutter("add store entry %r" % (fileid))
121
if isinstance(f, types.StringTypes):
126
p = self._path(fileid)
127
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
128
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
134
af = AtomicFile(fn, 'wb')
137
gf = gzip.GzipFile(mode='wb', fileobj=af)
147
def copy_multi(self, other, ids, permit_failure=False):
148
"""Copy texts for ids from other into self.
150
If an id is present in self, it is skipped.
152
Returns (count_copied, failed), where failed is a collection of ids
153
that could not be copied.
155
pb = bzrlib.ui.ui_factory.progress_bar()
157
pb.update('preparing to copy')
158
to_copy = [id for id in ids if id not in self]
159
if isinstance(other, ImmutableStore):
160
return self.copy_multi_immutable(other, to_copy, pb,
161
permit_failure=permit_failure)
166
pb.update('copy', count, len(to_copy))
167
if not permit_failure:
168
self.add(other[id], id)
177
if not permit_failure:
178
assert count == len(to_copy)
182
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
183
from shutil import copyfile
188
other_p = other._path(id)
192
if e.errno == errno.ENOENT:
193
if not permit_failure:
194
copyfile(other_p+".gz", p+".gz")
197
copyfile(other_p+".gz", p+".gz")
199
if e.errno == errno.ENOENT:
207
pb.update('copy', count, len(to_copy))
208
assert count == len(to_copy)
212
def __contains__(self, fileid):
214
p = self._path(fileid)
215
return (os.access(p, os.R_OK)
216
or os.access(p + '.gz', os.R_OK))
218
def _item_size(self, fid):
221
return os.stat(p)[ST_SIZE]
223
return os.stat(p + '.gz')[ST_SIZE]
226
for f in os.listdir(self._basedir):
228
# TODO: case-insensitive?
234
return len(os.listdir(self._basedir))
236
def __getitem__(self, fileid):
237
"""Returns a file reading from a particular entry."""
238
p = self._path(fileid)
240
return gzip.GzipFile(p + '.gz', 'rb')
242
if e.errno != errno.ENOENT:
248
if e.errno != errno.ENOENT:
251
raise KeyError(fileid)
254
class ImmutableScratchStore(ImmutableStore):
255
"""Self-destructing test subclass of ImmutableStore.
257
The Store only exists for the lifetime of the Python object.
258
Obviously you should not put anything precious in it.
261
super(ImmutableScratchStore, self).__init__(tempfile.mkdtemp())
264
for f in os.listdir(self._basedir):
265
fpath = os.path.join(self._basedir, f)
266
# needed on windows, and maybe some other filesystems
267
os.chmod(fpath, 0600)
269
os.rmdir(self._basedir)
270
mutter("%r destroyed" % self)
273
class ImmutableMemoryStore(Store):
274
"""A memory only store."""
277
super(ImmutableMemoryStore, self).__init__()
280
def add(self, stream, fileid, compressed=True):
281
if self._contents.has_key(fileid):
282
raise StoreError("fileid %s already in the store" % fileid)
283
self._contents[fileid] = stream.read()
285
def __getitem__(self, fileid):
286
"""Returns a file reading from a particular entry."""
287
if not self._contents.has_key(fileid):
289
return StringIO(self._contents[fileid])
291
def _item_size(self, fileid):
292
return len(self._contents[fileid])
295
return iter(self._contents.keys())
298
class RemoteStore(object):
300
def __init__(self, baseurl):
301
self._baseurl = baseurl
303
def _path(self, name):
305
raise ValueError('invalid store id', name)
306
return self._baseurl + '/' + name
308
def __getitem__(self, fileid):
309
p = self._path(fileid)
311
return get_url(p, compressed=True)
312
except urllib2.URLError:
315
return get_url(p, compressed=False)
316
except urllib2.URLError:
317
raise KeyError(fileid)
320
"""A store that caches data locally, to avoid repeated downloads.
321
The precacache method should be used to avoid server round-trips for
325
def __init__(self, store, cache_dir):
326
self.source_store = store
327
self.cache_store = ImmutableStore(cache_dir)
329
def __getitem__(self, id):
330
mutter("Cache add %s" % id)
331
if id not in self.cache_store:
332
self.cache_store.add(self.source_store[id], id)
333
return self.cache_store[id]
335
def prefetch(self, ids):
336
"""Copy a series of ids into the cache, before they are used.
337
For remote stores that support pipelining or async downloads, this can
338
increase speed considerably.
339
Failures while prefetching are ignored.
341
mutter("Prefetch of ids %s" % ",".join(ids))
342
self.cache_store.copy_multi(self.source_store, ids,
added
removed
bzrlib/store.py
