To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to cache.py
- browse files at revision 0.200.1028
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.1028
- Committer: Jelmer Vernooij
- Date: 2010-09-02 19:34:29 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100902193429-pg9u1cf8l7krkad2
Fix imports of TestLoader on newer versions of bzr.
- files added:
- help.py
- files renamed:
- shamap.py => cache.py
- tests/test_shamap.py => tests/test_cache.py
- files modified:
- Makefile
added
removed
cache.py
23
23
import os
24
24
import threading
25
25
26
from dulwich.objects import (
27
ShaFile,
28
)
29
26
30
import bzrlib
27
31
from bzrlib import (
32
btree_index as _mod_btree_index,
33
index as _mod_index,
34
osutils,
35
registry,
28
36
trace,
37
versionedfile,
38
)
39
from bzrlib.transport import (
40
get_transport,
29
41
)
30
42
31
43
42
54
return ret
43
55
44
56
57
def get_remote_cache_transport():
58
"""Retrieve the transport to use when accessing (unwritable) remote
59
repositories.
60
"""
61
return get_transport(get_cache_dir())
62
63
45
64
def check_pysqlite_version(sqlite3):
46
65
"""Check that sqlite library is compatible.
47
66
75
94
return _mapdbs.cache
76
95
77
96
78
class InventorySHAMap(object):
79
"""Maps inventory file ids to Git SHAs."""
80
81
def lookup_blob(self, file_id, revision):
97
class GitShaMap(object):
98
"""Git<->Bzr revision id mapping database."""
99
100
def lookup_git_sha(self, sha):
101
"""Lookup a Git sha in the database.
102
:param sha: Git object sha
103
:return: (type, type_data) with type_data:
104
revision: revid, tree sha
105
"""
106
raise NotImplementedError(self.lookup_git_sha)
107
108
def lookup_blob_id(self, file_id, revision):
82
109
"""Retrieve a Git blob SHA by file id.
83
110
84
111
:param file_id: File id of the file/symlink
85
112
:param revision: revision in which the file was last changed.
86
113
"""
87
raise NotImplementedError(self.lookup_blob)
114
raise NotImplementedError(self.lookup_blob_id)
88
115
89
def lookup_tree(self, file_id):
116
def lookup_tree_id(self, file_id, revision):
90
117
"""Retrieve a Git tree SHA by file id.
91
118
"""
92
raise NotImplementedError(self.lookup_tree)
93
94
95
class GitShaMap(object):
96
"""Git<->Bzr revision id mapping database."""
97
98
def _add_entry(self, sha, type, type_data):
99
"""Add a new entry to the database.
100
"""
101
raise NotImplementedError(self._add_entry)
102
103
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
104
entries):
105
"""Add multiple new entries to the database.
106
"""
107
for (fileid, kind, hexsha, revision) in entries:
108
self._add_entry(hexsha, kind, (fileid, revision))
109
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
110
111
def get_inventory_sha_map(self, revid):
112
"""Return the inventory SHA map for a revision.
113
114
:param revid: Revision to fetch the map for
115
:return: A `InventorySHAMap`
116
"""
117
raise NotImplementedError(self.get_inventory_sha_map)
118
119
def lookup_git_sha(self, sha):
120
"""Lookup a Git sha in the database.
121
:param sha: Git object sha
122
:return: (type, type_data) with type_data:
123
revision: revid, tree sha
124
"""
125
raise NotImplementedError(self.lookup_git_sha)
119
raise NotImplementedError(self.lookup_tree_id)
126
120
127
121
def revids(self):
128
122
"""List the revision ids known."""
149
143
"""Abort any pending changes."""
150
144
151
145
146
class ContentCache(object):
147
"""Object that can cache Git objects."""
148
149
def add(self, object):
150
"""Add an object."""
151
raise NotImplementedError(self.add)
152
153
def add_multi(self, objects):
154
"""Add multiple objects."""
155
for obj in objects:
156
self.add(obj)
157
158
def __getitem__(self, sha):
159
"""Retrieve an item, by SHA."""
160
raise NotImplementedError(self.__getitem__)
161
162
163
class BzrGitCacheFormat(object):
164
"""Bazaar-Git Cache Format."""
165
166
def get_format_string(self):
167
"""Return a single-line unique format string for this cache format."""
168
raise NotImplementedError(self.get_format_string)
169
170
def open(self, transport):
171
"""Open this format on a transport."""
172
raise NotImplementedError(self.open)
173
174
def initialize(self, transport):
175
"""Create a new instance of this cache format at transport."""
176
transport.put_bytes('format', self.get_format_string())
177
178
@classmethod
179
def from_transport(self, transport):
180
"""Open a cache file present on a transport, or initialize one.
181
182
:param transport: Transport to use
183
:return: A BzrGitCache instance
184
"""
185
try:
186
format_name = transport.get_bytes('format')
187
format = formats.get(format_name)
188
except bzrlib.errors.NoSuchFile:
189
format = formats.get('default')
190
format.initialize(transport)
191
return format.open(transport)
192
193
@classmethod
194
def from_repository(cls, repository):
195
"""Open a cache file for a repository.
196
197
This will use the repository's transport to store the cache file, or
198
use the users global cache directory if the repository has no
199
transport associated with it.
200
201
:param repository: Repository to open the cache for
202
:return: A `BzrGitCache`
203
"""
204
repo_transport = getattr(repository, "_transport", None)
205
if repo_transport is not None:
206
# Even if we don't write to this repo, we should be able
207
# to update its cache.
208
repo_transport = remove_readonly_transport_decorator(repo_transport)
209
try:
210
repo_transport.mkdir('git')
211
except bzrlib.errors.FileExists:
212
pass
213
transport = repo_transport.clone('git')
214
else:
215
transport = get_remote_cache_transport()
216
return cls.from_transport(transport)
217
218
219
class CacheUpdater(object):
220
"""Base class for objects that can update a bzr-git cache."""
221
222
def add_object(self, obj, ie, path):
223
raise NotImplementedError(self.add_object)
224
225
def finish(self):
226
raise NotImplementedError(self.finish)
227
228
229
class BzrGitCache(object):
230
"""Caching backend."""
231
232
def __init__(self, idmap, content_cache, cache_updater_klass):
233
self.idmap = idmap
234
self.content_cache = content_cache
235
self._cache_updater_klass = cache_updater_klass
236
237
def get_updater(self, rev):
238
"""Update an object that implements the CacheUpdater interface for
239
updating this cache.
240
"""
241
return self._cache_updater_klass(self, rev)
242
243
244
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), None, DictCacheUpdater)
245
246
247
class DictCacheUpdater(CacheUpdater):
248
"""Cache updater for dict-based caches."""
249
250
def __init__(self, cache, rev):
251
self.cache = cache
252
self.revid = rev.revision_id
253
self.parent_revids = rev.parent_ids
254
self._commit = None
255
self._entries = []
256
257
def add_object(self, obj, ie, path):
258
if obj.type_name == "commit":
259
self._commit = obj
260
assert ie is None
261
type_data = (self.revid, self._commit.tree)
262
self.cache.idmap._by_revid[self.revid] = obj.id
263
elif obj.type_name in ("blob", "tree"):
264
if ie is not None:
265
if obj.type_name == "blob":
266
revision = ie.revision
267
else:
268
revision = self.revid
269
type_data = (ie.file_id, revision)
270
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] =\
271
obj.id
272
else:
273
raise AssertionError
274
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
275
276
def finish(self):
277
if self._commit is None:
278
raise AssertionError("No commit object added")
279
return self._commit
280
281
152
282
class DictGitShaMap(GitShaMap):
283
"""Git SHA map that uses a dictionary."""
153
284
154
285
def __init__(self):
155
286
self._by_sha = {}
156
287
self._by_fileid = {}
157
158
def _add_entry(self, sha, type, type_data):
159
self._by_sha[sha] = (type, type_data)
160
if type in ("blob", "tree"):
161
self._by_fileid.setdefault(type_data[1], {})[type_data[0]] = sha
162
163
def get_inventory_sha_map(self, revid):
164
165
class DictInventorySHAMap(InventorySHAMap):
166
167
def __init__(self, base, revid):
168
self._base = base
169
self.revid = revid
170
171
def lookup_blob(self, fileid, revision):
172
return self._base._by_fileid[revision][fileid]
173
174
def lookup_tree(self, fileid):
175
return self._base._by_fileid[self.revid][fileid]
176
177
return DictInventorySHAMap(self, revid)
288
self._by_revid = {}
289
290
def lookup_blob_id(self, fileid, revision):
291
return self._by_fileid[revision][fileid]
178
292
179
293
def lookup_git_sha(self, sha):
180
294
return self._by_sha[sha]
181
295
296
def lookup_tree_id(self, fileid, revision):
297
return self._by_fileid[revision][fileid]
298
299
def lookup_commit(self, revid):
300
return self._by_revid[revid]
301
182
302
def revids(self):
183
303
for key, (type, type_data) in self._by_sha.iteritems():
184
304
if type == "commit":
188
308
return self._by_sha.iterkeys()
189
309
190
310
311
class SqliteCacheUpdater(CacheUpdater):
312
313
def __init__(self, cache, rev):
314
self.cache = cache
315
self.db = self.cache.idmap.db
316
self.revid = rev.revision_id
317
self._commit = None
318
self._trees = []
319
self._blobs = []
320
321
def add_object(self, obj, ie, path):
322
if obj.type_name == "commit":
323
self._commit = obj
324
assert ie is None
325
elif obj.type_name == "tree":
326
if ie is not None:
327
self._trees.append((obj.id, ie.file_id, self.revid))
328
elif obj.type_name == "blob":
329
if ie is not None:
330
self._blobs.append((obj.id, ie.file_id, ie.revision))
331
else:
332
raise AssertionError
333
334
def finish(self):
335
if self._commit is None:
336
raise AssertionError("No commit object added")
337
self.db.executemany(
338
"replace into trees (sha1, fileid, revid) values (?, ?, ?)",
339
self._trees)
340
self.db.executemany(
341
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
342
self._blobs)
343
self.db.execute(
344
"replace into commits (sha1, revid, tree_sha) values (?, ?, ?)",
345
(self._commit.id, self.revid, self._commit.tree))
346
return self._commit
347
348
349
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), None, SqliteCacheUpdater)
350
351
352
class SqliteGitCacheFormat(BzrGitCacheFormat):
353
354
def get_format_string(self):
355
return 'bzr-git sha map version 1 using sqlite\n'
356
357
def open(self, transport):
358
try:
359
basepath = transport.local_abspath(".")
360
except bzrlib.errors.NotLocalUrl:
361
basepath = get_cache_dir()
362
return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))
363
364
191
365
class SqliteGitShaMap(GitShaMap):
366
"""Bazaar GIT Sha map that uses a sqlite database for storage."""
192
367
193
368
def __init__(self, path=None):
194
369
self.path = path
226
401
def __repr__(self):
227
402
return "%s(%r)" % (self.__class__.__name__, self.path)
228
403
229
@classmethod
230
def from_repository(cls, repository):
231
try:
232
transport = getattr(repository, "_transport", None)
233
if transport is not None:
234
return cls(os.path.join(transport.local_abspath("."), "git.db"))
235
except bzrlib.errors.NotLocalUrl:
236
pass
237
return cls(os.path.join(get_cache_dir(), "remote.db"))
238
239
404
def lookup_commit(self, revid):
240
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
405
cursor = self.db.execute("select sha1 from commits where revid = ?",
406
(revid,))
407
row = cursor.fetchone()
241
408
if row is not None:
242
409
return row[0]
243
410
raise KeyError
245
412
def commit_write_group(self):
246
413
self.db.commit()
247
414
248
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
249
entries):
250
trees = []
251
blobs = []
252
for (fileid, kind, hexsha, revision) in entries:
253
if kind is None:
254
continue
255
if kind == "tree":
256
trees.append((hexsha, fileid, revid))
257
elif kind == "blob":
258
blobs.append((hexsha, fileid, revision))
259
else:
260
raise AssertionError
261
if trees:
262
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
263
if blobs:
264
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
265
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
266
267
def _add_entry(self, sha, type, type_data):
268
"""Add a new entry to the database.
269
"""
270
assert isinstance(type_data, tuple)
271
if sha is None:
272
return
273
assert isinstance(sha, str), "type was %r" % sha
274
if type == "commit":
275
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
276
elif type in ("blob", "tree"):
277
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
278
else:
279
raise AssertionError("Unknown type %s" % type)
280
281
def get_inventory_sha_map(self, revid):
282
class SqliteInventorySHAMap(InventorySHAMap):
283
284
def __init__(self, db, revid):
285
self.db = db
286
self.revid = revid
287
288
def lookup_blob(self, fileid, revision):
289
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
290
if row is not None:
291
return row[0]
292
raise KeyError(fileid)
293
294
def lookup_tree(self, fileid):
295
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, self.revid)).fetchone()
296
if row is not None:
297
return row[0]
298
raise KeyError(fileid)
299
300
return SqliteInventorySHAMap(self.db, revid)
415
def lookup_blob_id(self, fileid, revision):
416
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
417
if row is not None:
418
return row[0]
419
raise KeyError(fileid)
420
421
def lookup_tree_id(self, fileid, revision):
422
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
423
if row is not None:
424
return row[0]
425
raise KeyError(fileid)
301
426
302
427
def lookup_git_sha(self, sha):
303
428
"""Lookup a Git sha in the database.
306
431
:return: (type, type_data) with type_data:
307
432
revision: revid, tree sha
308
433
"""
309
def format(type, row):
310
return (type, (row[0], row[1]))
311
434
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
312
435
if row is not None:
313
return format("commit", row)
436
return ("commit", row)
314
437
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
315
438
if row is not None:
316
return format("blob", row)
439
return ("blob", row)
317
440
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
318
441
if row is not None:
319
return format("tree", row)
442
return ("tree", row)
320
443
raise KeyError(sha)
321
444
322
445
def revids(self):
326
449
def sha1s(self):
327
450
"""List the SHA1s."""
328
451
for table in ("blobs", "commits", "trees"):
329
trace.note(table)
330
for (row,) in self.db.execute("select sha1 from %s" % table):
331
yield row
332
333
334
TDB_MAP_VERSION = 3
335
TDB_HASH_SIZE = 50000
452
for (sha,) in self.db.execute("select sha1 from %s" % table):
453
yield sha
454
455
456
class TdbCacheUpdater(CacheUpdater):
457
"""Cache updater for tdb-based caches."""
458
459
def __init__(self, cache, rev):
460
self.cache = cache
461
self.db = cache.idmap.db
462
self.revid = rev.revision_id
463
self.parent_revids = rev.parent_ids
464
self._commit = None
465
self._entries = []
466
467
def add_object(self, obj, ie, path):
468
sha = obj.sha().digest()
469
if obj.type_name == "commit":
470
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
471
type_data = (self.revid, obj.tree)
472
self._commit = obj
473
assert ie is None
474
elif obj.type_name == "blob":
475
if ie is None:
476
return
477
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
478
type_data = (ie.file_id, ie.revision)
479
elif obj.type_name == "tree":
480
if ie is None:
481
return
482
type_data = (ie.file_id, self.revid)
483
else:
484
raise AssertionError
485
self.db["git\0" + sha] = "\0".join((obj.type_name, ) + type_data)
486
487
def finish(self):
488
if self._commit is None:
489
raise AssertionError("No commit object added")
490
return self._commit
491
492
493
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
494
495
class TdbGitCacheFormat(BzrGitCacheFormat):
496
"""Cache format for tdb-based caches."""
497
498
def get_format_string(self):
499
return 'bzr-git sha map version 3 using tdb\n'
500
501
def open(self, transport):
502
try:
503
basepath = transport.local_abspath(".")
504
except bzrlib.errors.NotLocalUrl:
505
basepath = get_cache_dir()
506
try:
507
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
508
except ImportError:
509
raise ImportError(
510
"Unable to open existing bzr-git cache because 'tdb' is not "
511
"installed.")
336
512
337
513
338
514
class TdbGitShaMap(GitShaMap):
346
522
"blob fileid revid" -> "<sha1>"
347
523
"""
348
524
525
TDB_MAP_VERSION = 3
526
TDB_HASH_SIZE = 50000
527
349
528
def __init__(self, path=None):
350
529
import tdb
351
530
self.path = path
353
532
self.db = {}
354
533
else:
355
534
if not mapdbs().has_key(path):
356
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
535
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
357
536
os.O_RDWR|os.O_CREAT)
358
537
self.db = mapdbs()[path]
359
538
try:
360
539
if int(self.db["version"]) not in (2, 3):
361
540
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
362
self.db["version"], TDB_MAP_VERSION)
541
self.db["version"], self.TDB_MAP_VERSION)
363
542
self.db.clear()
364
543
except KeyError:
365
544
pass
366
self.db["version"] = str(TDB_MAP_VERSION)
545
self.db["version"] = str(self.TDB_MAP_VERSION)
367
546
368
547
def start_write_group(self):
369
548
"""Start writing changes."""
380
559
def __repr__(self):
381
560
return "%s(%r)" % (self.__class__.__name__, self.path)
382
561
383
@classmethod
384
def from_repository(cls, repository):
385
try:
386
transport = getattr(repository, "_transport", None)
387
if transport is not None:
388
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
389
except bzrlib.errors.NotLocalUrl:
390
pass
391
return cls(os.path.join(get_cache_dir(), "remote.tdb"))
392
393
562
def lookup_commit(self, revid):
394
563
return sha_to_hex(self.db["commit\0" + revid][:20])
395
564
396
def _add_entry(self, hexsha, type, type_data):
397
"""Add a new entry to the database.
398
"""
399
if hexsha is None:
400
sha = ""
401
else:
402
sha = hex_to_sha(hexsha)
403
self.db["git\0" + sha] = "\0".join((type, type_data[0], type_data[1]))
404
if type == "commit":
405
self.db["commit\0" + type_data[0]] = "\0".join((sha, type_data[1]))
406
elif type == "blob":
407
self.db["\0".join(("blob", type_data[0], type_data[1]))] = sha
408
409
def get_inventory_sha_map(self, revid):
410
411
class TdbInventorySHAMap(InventorySHAMap):
412
413
def __init__(self, db, revid):
414
self.db = db
415
self.revid = revid
416
417
def lookup_blob(self, fileid, revision):
418
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
565
def lookup_blob_id(self, fileid, revision):
566
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
419
567
420
return TdbInventorySHAMap(self.db, revid)
421
422
568
def lookup_git_sha(self, sha):
423
569
"""Lookup a Git sha in the database.
424
570
451
597
yield sha_to_hex(key[4:])
452
598
453
599
600
class VersionedFilesContentCache(ContentCache):
601
602
def __init__(self, vf):
603
self._vf = vf
604
605
def add(self, obj):
606
self._vf.insert_record_stream(
607
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
608
obj.as_legacy_object_chunks())])
609
610
def __getitem__(self, sha):
611
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
612
entry = stream.next()
613
if entry.storage_kind == 'absent':
614
raise KeyError(sha)
615
return ShaFile._parse_legacy_object(entry.get_bytes_as('fulltext'))
616
617
618
class GitObjectStoreContentCache(ContentCache):
619
620
def __init__(self, store):
621
self.store = store
622
623
def add_multi(self, objs):
624
self.store.add_objects(objs)
625
626
def add(self, obj, path):
627
self.store.add_object(obj)
628
629
def __getitem__(self, sha):
630
return self.store[sha]
631
632
633
class IndexCacheUpdater(CacheUpdater):
634
635
def __init__(self, cache, rev):
636
self.cache = cache
637
self.revid = rev.revision_id
638
self.parent_revids = rev.parent_ids
639
self._commit = None
640
self._entries = []
641
self._cache_objs = set()
642
643
def add_object(self, obj, ie, path):
644
if obj.type_name == "commit":
645
self._commit = obj
646
assert ie is None
647
self.cache.idmap._add_git_sha(obj.id, "commit",
648
(self.revid, obj.tree))
649
self.cache.idmap._add_node(("commit", self.revid, "X"),
650
" ".join((obj.id, obj.tree)))
651
self._cache_objs.add((obj, path))
652
elif obj.type_name == "blob":
653
self.cache.idmap._add_git_sha(obj.id, "blob",
654
(ie.file_id, ie.revision))
655
self.cache.idmap._add_node(("blob", ie.file_id, ie.revision), obj.id)
656
if ie.kind == "symlink":
657
self._cache_objs.add((obj, path))
658
elif obj.type_name == "tree":
659
self.cache.idmap._add_git_sha(obj.id, "tree",
660
(ie.file_id, self.revid))
661
self._cache_objs.add((obj, path))
662
else:
663
raise AssertionError
664
665
def finish(self):
666
self.cache.content_cache.add_multi(self._cache_objs)
667
return self._commit
668
669
670
class IndexBzrGitCache(BzrGitCache):
671
672
def __init__(self, transport=None):
673
mapper = versionedfile.ConstantMapper("trees")
674
shamap = IndexGitShaMap(transport.clone('index'))
675
#trees_store = knit.make_file_factory(True, mapper)(transport)
676
#content_cache = VersionedFilesContentCache(trees_store)
677
from bzrlib.plugins.git.transportgit import TransportObjectStore
678
store = TransportObjectStore(transport.clone('objects'))
679
content_cache = GitObjectStoreContentCache(store)
680
super(IndexBzrGitCache, self).__init__(shamap, content_cache,
681
IndexCacheUpdater)
682
683
684
class IndexGitCacheFormat(BzrGitCacheFormat):
685
686
def get_format_string(self):
687
return 'bzr-git sha map with git object cache version 1\n'
688
689
def initialize(self, transport):
690
super(IndexGitCacheFormat, self).initialize(transport)
691
transport.mkdir('index')
692
transport.mkdir('objects')
693
from bzrlib.plugins.git.transportgit import TransportObjectStore
694
TransportObjectStore.init(transport.clone('objects'))
695
696
def open(self, transport):
697
return IndexBzrGitCache(transport)
698
699
700
class IndexGitShaMap(GitShaMap):
701
"""SHA Map that uses the Bazaar APIs to store a cache.
702
703
BTree Index file with the following contents:
704
705
("git", <sha1>) -> "<type> <type-data1> <type-data2>"
706
("commit", <revid>) -> "<sha1> <tree-id>"
707
("blob", <fileid>, <revid>) -> <sha1>
708
709
"""
710
711
def __init__(self, transport=None):
712
if transport is None:
713
self._transport = None
714
self._index = _mod_index.InMemoryGraphIndex(0, key_elements=3)
715
self._builder = self._index
716
else:
717
self._builder = None
718
self._transport = transport
719
self._index = _mod_index.CombinedGraphIndex([])
720
for name in self._transport.list_dir("."):
721
if not name.endswith(".rix"):
722
continue
723
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
724
self._transport.stat(name).st_size)
725
self._index.insert_index(0, x)
726
727
@classmethod
728
def from_repository(cls, repository):
729
transport = getattr(repository, "_transport", None)
730
if transport is not None:
731
try:
732
transport.mkdir('git')
733
except bzrlib.errors.FileExists:
734
pass
735
return cls(transport.clone('git'))
736
from bzrlib.transport import get_transport
737
return cls(get_transport(get_cache_dir()))
738
739
def __repr__(self):
740
if self._transport is not None:
741
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
742
else:
743
return "%s()" % (self.__class__.__name__)
744
745
def repack(self):
746
assert self._builder is None
747
self.start_write_group()
748
for _, key, value in self._index.iter_all_entries():
749
self._builder.add_node(key, value)
750
to_remove = []
751
for name in self._transport.list_dir('.'):
752
if name.endswith('.rix'):
753
to_remove.append(name)
754
self.commit_write_group()
755
del self._index.indices[1:]
756
for name in to_remove:
757
self._transport.rename(name, name + '.old')
758
759
def start_write_group(self):
760
assert self._builder is None
761
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
762
self._name = osutils.sha()
763
764
def commit_write_group(self):
765
assert self._builder is not None
766
stream = self._builder.finish()
767
name = self._name.hexdigest() + ".rix"
768
size = self._transport.put_file(name, stream)
769
index = _mod_btree_index.BTreeGraphIndex(self._transport, name, size)
770
self._index.insert_index(0, index)
771
self._builder = None
772
self._name = None
773
774
def abort_write_group(self):
775
assert self._builder is not None
776
self._builder = None
777
self._name = None
778
779
def _add_node(self, key, value):
780
try:
781
self._builder.add_node(key, value)
782
except bzrlib.errors.BadIndexDuplicateKey:
783
# Multiple bzr objects can have the same contents
784
return True
785
else:
786
return False
787
788
def _get_entry(self, key):
789
entries = self._index.iter_entries([key])
790
try:
791
return entries.next()[2]
792
except StopIteration:
793
if self._builder is None:
794
raise KeyError
795
entries = self._builder.iter_entries([key])
796
try:
797
return entries.next()[2]
798
except StopIteration:
799
raise KeyError
800
801
def _iter_keys_prefix(self, prefix):
802
for entry in self._index.iter_entries_prefix([prefix]):
803
yield entry[1]
804
if self._builder is not None:
805
for entry in self._builder.iter_entries_prefix([prefix]):
806
yield entry[1]
807
808
def lookup_commit(self, revid):
809
return self._get_entry(("commit", revid, "X"))[:40]
810
811
def _add_git_sha(self, hexsha, type, type_data):
812
if hexsha is not None:
813
self._name.update(hexsha)
814
self._add_node(("git", hexsha, "X"),
815
" ".join((type, type_data[0], type_data[1])))
816
else:
817
# This object is not represented in Git - perhaps an empty
818
# directory?
819
self._name.update(type + " ".join(type_data))
820
821
def lookup_blob_id(self, fileid, revision):
822
return self._get_entry(("blob", fileid, revision))
823
824
def lookup_git_sha(self, sha):
825
if len(sha) == 20:
826
sha = sha_to_hex(sha)
827
data = self._get_entry(("git", sha, "X")).split(" ", 2)
828
return (data[0], (data[1], data[2]))
829
830
def revids(self):
831
"""List the revision ids known."""
832
for key in self._iter_keys_prefix(("commit", None, None)):
833
yield key[1]
834
835
def missing_revisions(self, revids):
836
"""Return set of all the revisions that are not present."""
837
missing_revids = set(revids)
838
for _, key, value in self._index.iter_entries((
839
("commit", revid, "X") for revid in revids)):
840
missing_revids.remove(key[1])
841
return missing_revids
842
843
def sha1s(self):
844
"""List the SHA1s."""
845
for key in self._iter_keys_prefix(("git", None, None)):
846
yield key[1]
847
848
849
formats = registry.Registry()
850
formats.register(TdbGitCacheFormat().get_format_string(),
851
TdbGitCacheFormat())
852
formats.register(SqliteGitCacheFormat().get_format_string(),
853
SqliteGitCacheFormat())
854
formats.register(IndexGitCacheFormat().get_format_string(),
855
IndexGitCacheFormat())
856
# In the future, this will become the default:
857
# formats.register('default', IndexGitCacheFormat())
858
try:
859
import tdb
860
except ImportError:
861
formats.register('default', SqliteGitCacheFormat())
862
else:
863
formats.register('default', TdbGitCacheFormat())
864
865
866
867
def migrate_ancient_formats(repo_transport):
868
# Prefer migrating git.db over git.tdb, since the latter may not
869
# be openable on some platforms.
870
if repo_transport.has("git.db"):
871
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
872
repo_transport.rename("git.db", "git/idmap.db")
873
elif repo_transport.has("git.tdb"):
874
TdbGitCacheFormat().initialize(repo_transport.clone("git"))
875
repo_transport.rename("git.tdb", "git/idmap.tdb")
876
877
878
def remove_readonly_transport_decorator(transport):
879
if transport.is_readonly():
880
return transport._decorated
881
return transport
882
883
454
884
def from_repository(repository):
455
try:
456
return TdbGitShaMap.from_repository(repository)
457
except ImportError:
458
return SqliteGitShaMap.from_repository(repository)
885
"""Open a cache file for a repository.
886
887
If the repository is remote and there is no transport available from it
888
this will use a local file in the users cache directory
889
(typically ~/.cache/bazaar/git/)
890
891
:param repository: A repository object
892
"""
893
repo_transport = getattr(repository, "_transport", None)
894
if repo_transport is not None:
895
# Migrate older cache formats
896
repo_transport = remove_readonly_transport_decorator(repo_transport)
897
try:
898
repo_transport.mkdir("git")
899
except bzrlib.errors.FileExists:
900
pass
901
else:
902
migrate_ancient_formats(repo_transport)
903
return BzrGitCacheFormat.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0