1
# Copyright (C) 2005 by Canonical Development Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
Stores are the main data-storage mechanism for Bazaar-NG.
20
A store is a simple write-once container indexed by a universally
29
from stat import ST_SIZE
30
from StringIO import StringIO
32
from bzrlib.errors import BzrError, UnlistableStore
33
from bzrlib.trace import mutter
35
import bzrlib.osutils as osutils
37
#from bzrlib.remotebranch import get_url
41
######################################################################
44
class StoreError(Exception):
49
"""An abstract store that holds files indexed by unique names.
51
Files can be added, but not modified once they are in. Typically
52
the hash is used as the name, or something else known to be unique,
55
>>> st = ImmutableScratchStore()
57
>>> st.add(StringIO('hello'), 'aa')
63
You are not allowed to add an id that is already present.
65
Entries can be retrieved as files, which may then be read.
67
>>> st.add(StringIO('goodbye'), '123123')
68
>>> st['123123'].read()
73
"""Return (count, bytes)
75
This is the (compressed) size stored on disk, not the size of
81
total += self._item_size(fid)
85
class ImmutableStore(Store):
86
"""Store that stores files on disk.
88
TODO: Atomic add by writing to a temporary file and renaming.
89
TODO: Guard against the same thing being stored twice, compressed and
90
uncompressed during copy_multi_immutable - the window is for a
91
matching store with some crack code that lets it offer a
92
non gz FOO and then a fz FOO.
94
In bzr 0.0.5 and earlier, files within the store were marked
95
readonly on disk. This is no longer done but existing stores need
99
def __init__(self, basedir):
100
super(ImmutableStore, self).__init__()
101
self._basedir = basedir
103
def _path(self, entry_id):
104
if not isinstance(entry_id, basestring):
105
raise TypeError(type(entry_id))
106
if '\\' in entry_id or '/' in entry_id:
107
raise ValueError("invalid store id %r" % entry_id)
108
return os.path.join(self._basedir, entry_id)
111
return "%s(%r)" % (self.__class__.__name__, self._basedir)
113
def add(self, f, fileid, compressed=True):
114
"""Add contents of a file into the store.
116
f -- An open file, or file-like object."""
117
# FIXME: Only works on files that will fit in memory
119
from bzrlib.atomicfile import AtomicFile
121
mutter("add store entry %r" % (fileid))
122
if isinstance(f, types.StringTypes):
127
p = self._path(fileid)
128
if os.access(p, os.F_OK) or os.access(p + '.gz', os.F_OK):
129
raise BzrError("store %r already contains id %r" % (self._basedir, fileid))
135
af = AtomicFile(fn, 'wb')
138
gf = gzip.GzipFile(mode='wb', fileobj=af)
148
def copy_multi(self, other, ids, permit_failure=False):
149
"""Copy texts for ids from other into self.
151
If an id is present in self, it is skipped.
153
Returns (count_copied, failed), where failed is a collection of ids
154
that could not be copied.
156
pb = bzrlib.ui.ui_factory.progress_bar()
158
pb.update('preparing to copy')
159
to_copy = [id for id in ids if id not in self]
160
if isinstance(other, ImmutableStore):
161
return self.copy_multi_immutable(other, to_copy, pb,
162
permit_failure=permit_failure)
167
pb.update('copy', count, len(to_copy))
168
if not permit_failure:
169
self.add(other[id], id)
178
if not permit_failure:
179
assert count == len(to_copy)
183
def copy_multi_immutable(self, other, to_copy, pb, permit_failure=False):
188
other_p = other._path(id)
190
osutils.link_or_copy(other_p, p)
191
except (IOError, OSError), e:
192
if e.errno == errno.ENOENT:
193
if not permit_failure:
194
osutils.link_or_copy(other_p+".gz", p+".gz")
197
osutils.link_or_copy(other_p+".gz", p+".gz")
199
if e.errno == errno.ENOENT:
207
pb.update('copy', count, len(to_copy))
208
assert count == len(to_copy)
212
def __contains__(self, fileid):
214
p = self._path(fileid)
215
return (os.access(p, os.R_OK)
216
or os.access(p + '.gz', os.R_OK))
218
def _item_size(self, fid):
221
return os.stat(p)[ST_SIZE]
223
return os.stat(p + '.gz')[ST_SIZE]
226
for f in os.listdir(self._basedir):
228
# TODO: case-insensitive?
234
return len(os.listdir(self._basedir))
236
def __getitem__(self, fileid):
237
"""Returns a file reading from a particular entry."""
238
p = self._path(fileid)
240
return gzip.GzipFile(p + '.gz', 'rb')
242
if e.errno != errno.ENOENT:
248
if e.errno != errno.ENOENT:
251
raise KeyError(fileid)
254
class ImmutableScratchStore(ImmutableStore):
255
"""Self-destructing test subclass of ImmutableStore.
257
The Store only exists for the lifetime of the Python object.
258
Obviously you should not put anything precious in it.
261
super(ImmutableScratchStore, self).__init__(tempfile.mkdtemp())
264
for f in os.listdir(self._basedir):
265
fpath = os.path.join(self._basedir, f)
266
# needed on windows, and maybe some other filesystems
267
os.chmod(fpath, 0600)
269
os.rmdir(self._basedir)
270
mutter("%r destroyed" % self)
273
class ImmutableMemoryStore(Store):
274
"""A memory only store."""
277
super(ImmutableMemoryStore, self).__init__()
280
def add(self, stream, fileid, compressed=True):
281
if self._contents.has_key(fileid):
282
raise StoreError("fileid %s already in the store" % fileid)
283
self._contents[fileid] = stream.read()
285
def __getitem__(self, fileid):
286
"""Returns a file reading from a particular entry."""
287
if not self._contents.has_key(fileid):
289
return StringIO(self._contents[fileid])
291
def _item_size(self, fileid):
292
return len(self._contents[fileid])
295
return iter(self._contents.keys())
298
class RemoteStore(object):
300
def __init__(self, baseurl):
301
self._baseurl = baseurl
303
def _path(self, name):
305
raise ValueError('invalid store id', name)
306
return self._baseurl + '/' + name
308
def __getitem__(self, fileid):
310
from bzrlib.remotebranch import get_url
311
p = self._path(fileid)
313
return get_url(p, compressed=True)
314
except urllib2.URLError:
317
return get_url(p, compressed=False)
318
except urllib2.URLError:
319
raise KeyError(fileid)
323
"""A store that caches data locally, to avoid repeated downloads.
324
The precacache method should be used to avoid server round-trips for
328
def __init__(self, store, cache_dir):
329
self.source_store = store
330
self.cache_store = ImmutableStore(cache_dir)
332
def __getitem__(self, id):
333
mutter("Cache add %s" % id)
334
if id not in self.cache_store:
335
self.cache_store.add(self.source_store[id], id)
336
return self.cache_store[id]
338
def prefetch(self, ids):
339
"""Copy a series of ids into the cache, before they are used.
340
For remote stores that support pipelining or async downloads, this can
341
increase speed considerably.
342
Failures while prefetching are ignored.
344
mutter("Prefetch of ids %s" % ",".join(ids))
345
self.cache_store.copy_multi(self.source_store, ids,
349
def copy_all(store_from, store_to):
350
"""Copy all ids from one store to another."""
351
if not hasattr(store_from, "__iter__"):
352
raise UnlistableStore(store_from)
353
ids = [f for f in store_from]
354
store_to.copy_multi(store_from, ids)
added
removed
bzrlib/store.py
