1
# Copyright (C) 2005 by Canonical Development Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
Stores are the main data-storage mechanism for Bazaar-NG.
20
A store is a simple write-once container indexed by a universally
24
import os, tempfile, types, osutils, gzip, errno
25
from stat import ST_SIZE
26
from StringIO import StringIO
27
from bzrlib.errors import BzrError
28
from bzrlib.trace import mutter
30
from bzrlib.remotebranch import get_url
32
######################################################################
35
class StoreError(Exception):
40
"""An abstract store that holds files indexed by unique names.
42
Files can be added, but not modified once they are in. Typically
43
the hash is used as the name, or something else known to be unique,
46
>>> st = ImmutableScratchStore()
48
>>> st.add(StringIO('hello'), 'aa')
54
You are not allowed to add an id that is already present.
56
Entries can be retrieved as files, which may then be read.
58
>>> st.add(StringIO('goodbye'), '123123')
59
>>> st['123123'].read()
64
"""Return (count, bytes)
66
This is the (compressed) size stored on disk, not the size of
72
total += self._item_size(fid)
76
class ImmutableStore(Store):
77
"""Store that stores files on disk.
79
TODO: Atomic add by writing to a temporary file and renaming.
80
TODO: Guard against the same thing being stored twice, compressed and
81
uncompressed during copy_multi_immutable - the window is for a
82
matching store with some crack code that lets it offer a
83
non gz FOO and then a fz FOO.
85
In bzr 0.0.5 and earlier, files within the store were marked
86
readonly on disk. This is no longer done but existing stores need
90
def __init__(self, basedir):
91
super(ImmutableStore, self).__init__()
92
self._basedir = basedir
95
if '\\' in id or '/' in id:
96
raise ValueError("invalid store id %r" % id)
97
return os.path.join(self._basedir, id)
100
return "%s(%r)" % (self.__class__.__name__, self._basedir)
102
def add(self, f, fileid, compressed=True):
103
"""Add contents of a file into the store.
105
f -- An open file, or file-like object."""
106
# FIXME: Only works on files that will fit in memory
108
from bzrlib.atomicfile import AtomicFile
110
mutter("add store entry %r" % (fileid))
111
if isinstance(f, types.StringTypes):
116
p = self._path(fileid)
117
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
118
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
124
af = AtomicFile(fn, 'wb')
127
gf = gzip.GzipFile(mode='wb', fileobj=af)
137
def copy_multi(self, other, ids, permit_failure=False):
138
"""Copy texts for ids from other into self.
140
If an id is present in self, it is skipped.
142
Returns (count_copied, failed), where failed is a collection of ids
143
that could not be copied.
145
pb = bzrlib.ui.ui_factory.progress_bar()
147
pb.update('preparing to copy')
148
to_copy = [id for id in ids if id not in self]
149
if isinstance(other, ImmutableStore):
150
return self.copy_multi_immutable(other, to_copy, pb)
155
pb.update('copy', count, len(to_copy))
156
if not permit_failure:
157
self.add(other[id], id)
166
if not permit_failure:
167
assert count == len(to_copy)
171
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
172
from shutil import copyfile
177
other_p = other._path(id)
181
if e.errno == errno.ENOENT:
182
if not permit_failure:
183
copyfile(other_p+".gz", p+".gz")
186
copyfile(other_p+".gz", p+".gz")
188
if e.errno == errno.ENOENT:
196
pb.update('copy', count, len(to_copy))
197
assert count == len(to_copy)
201
def __contains__(self, fileid):
203
p = self._path(fileid)
204
return (os.access(p, os.R_OK)
205
or os.access(p + '.gz', os.R_OK))
207
def _item_size(self, fid):
210
return os.stat(p)[ST_SIZE]
212
return os.stat(p + '.gz')[ST_SIZE]
215
for f in os.listdir(self._basedir):
217
# TODO: case-insensitive?
223
return len(os.listdir(self._basedir))
225
def __getitem__(self, fileid):
226
"""Returns a file reading from a particular entry."""
227
p = self._path(fileid)
229
return gzip.GzipFile(p + '.gz', 'rb')
231
if e.errno != errno.ENOENT:
237
if e.errno != errno.ENOENT:
240
raise IndexError(fileid)
243
class ImmutableScratchStore(ImmutableStore):
244
"""Self-destructing test subclass of ImmutableStore.
246
The Store only exists for the lifetime of the Python object.
247
Obviously you should not put anything precious in it.
250
super(ImmutableScratchStore, self).__init__(tempfile.mkdtemp())
253
for f in os.listdir(self._basedir):
254
fpath = os.path.join(self._basedir, f)
255
# needed on windows, and maybe some other filesystems
256
os.chmod(fpath, 0600)
258
os.rmdir(self._basedir)
259
mutter("%r destroyed" % self)
262
class ImmutableMemoryStore(Store):
263
"""A memory only store."""
266
super(ImmutableMemoryStore, self).__init__()
269
def add(self, stream, fileid, compressed=True):
270
if self._contents.has_key(fileid):
271
raise StoreError("fileid %s already in the store" % fileid)
272
self._contents[fileid] = stream.read()
274
def __getitem__(self, fileid):
275
"""Returns a file reading from a particular entry."""
276
if not self._contents.has_key(fileid):
278
return StringIO(self._contents[fileid])
280
def _item_size(self, fileid):
281
return len(self._contents[fileid])
284
return iter(self._contents.keys())
287
class RemoteStore(object):
289
def __init__(self, baseurl):
290
self._baseurl = baseurl
292
def _path(self, name):
294
raise ValueError('invalid store id', name)
295
return self._baseurl + '/' + name
297
def __getitem__(self, fileid):
298
p = self._path(fileid)
300
return get_url(p, compressed=True)
302
raise KeyError(fileid)
306
"""A store that caches data locally, to avoid repeated downloads.
307
The precacache method should be used to avoid server round-trips for
311
def __init__(self, store, cache_dir):
312
self.source_store = store
313
self.cache_store = ImmutableStore(cache_dir)
315
def __getitem__(self, id):
316
mutter("Cache add %s" % id)
317
if id not in self.cache_store:
318
self.cache_store.add(self.source_store[id], id)
319
return self.cache_store[id]
321
def prefetch(self, ids):
322
"""Copy a series of ids into the cache, before they are used.
323
For remote stores that support pipelining or async downloads, this can
324
increase speed considerably.
326
mutter("Prefetch of ids %s" % ",".join(ids))
327
self.cache_store.copy_multi(self.source_store, ids)
added
removed
bzrlib/store.py
