To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to cache.py
- browse files at revision 0.200.972
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.972
- Committer: Jelmer Vernooij
- Date: 2010-07-28 15:09:04 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100728150904-2ghxu1nmd57t7ii0
Use BZR_PLUGINS_AT.
- files added:
- notes/git-serve.txt
- files renamed:
- shamap.py => cache.py
- tests/test_shamap.py => tests/test_cache.py
- files modified:
- Makefile
added
removed
cache.py
23
23
import os
24
24
import threading
25
25
26
from dulwich.objects import (
27
ShaFile,
28
)
29
26
30
import bzrlib
27
31
from bzrlib import (
32
btree_index as _mod_btree_index,
33
index as _mod_index,
34
osutils,
35
registry,
28
36
trace,
37
versionedfile,
38
)
39
from bzrlib.transport import (
40
get_transport,
29
41
)
30
42
31
43
42
54
return ret
43
55
44
56
57
def get_remote_cache_transport():
58
return get_transport(get_cache_dir())
59
60
45
61
def check_pysqlite_version(sqlite3):
46
62
"""Check that sqlite library is compatible.
47
63
78
94
class GitShaMap(object):
79
95
"""Git<->Bzr revision id mapping database."""
80
96
81
def add_entry(self, sha, type, type_data):
82
"""Add a new entry to the database.
83
"""
84
raise NotImplementedError(self.add_entry)
85
86
def add_entries(self, entries):
87
"""Add multiple new entries to the database.
88
"""
89
for e in entries:
90
self.add_entry(*e)
91
92
def lookup_tree(self, fileid, revid):
93
"""Lookup the SHA of a git tree."""
94
raise NotImplementedError(self.lookup_tree)
95
96
def lookup_blob(self, fileid, revid):
97
"""Lookup a blob by the fileid it has in a bzr revision."""
98
raise NotImplementedError(self.lookup_blob)
99
100
97
def lookup_git_sha(self, sha):
101
98
"""Lookup a Git sha in the database.
102
103
99
:param sha: Git object sha
104
100
:return: (type, type_data) with type_data:
105
101
revision: revid, tree sha
106
102
"""
107
103
raise NotImplementedError(self.lookup_git_sha)
108
104
105
def lookup_blob_id(self, file_id, revision):
106
"""Retrieve a Git blob SHA by file id.
107
108
:param file_id: File id of the file/symlink
109
:param revision: revision in which the file was last changed.
110
"""
111
raise NotImplementedError(self.lookup_blob_id)
112
113
def lookup_tree_id(self, file_id, revision):
114
"""Retrieve a Git tree SHA by file id.
115
"""
116
raise NotImplementedError(self.lookup_tree_id)
117
109
118
def revids(self):
110
119
"""List the revision ids known."""
111
120
raise NotImplementedError(self.revids)
131
140
"""Abort any pending changes."""
132
141
133
142
143
class ContentCache(object):
144
"""Object that can cache Git objects."""
145
146
def add(self, object):
147
"""Add an object."""
148
raise NotImplementedError(self.add)
149
150
def add_multi(self, objects):
151
"""Add multiple objects."""
152
for obj in objects:
153
self.add(obj)
154
155
def __getitem__(self, sha):
156
"""Retrieve an item, by SHA."""
157
raise NotImplementedError(self.__getitem__)
158
159
160
class BzrGitCacheFormat(object):
161
"""Bazaar-Git Cache Format."""
162
163
def get_format_string(self):
164
"""Return a single-line unique format string for this cache format."""
165
raise NotImplementedError(self.get_format_string)
166
167
def open(self, transport):
168
"""Open this format on a transport."""
169
raise NotImplementedError(self.open)
170
171
def initialize(self, transport):
172
"""Create a new instance of this cache format at transport."""
173
transport.put_bytes('format', self.get_format_string())
174
175
@classmethod
176
def from_transport(self, transport):
177
"""Open a cache file present on a transport, or initialize one.
178
179
:param transport: Transport to use
180
:return: A BzrGitCache instance
181
"""
182
try:
183
format_name = transport.get_bytes('format')
184
format = formats.get(format_name)
185
except bzrlib.errors.NoSuchFile:
186
format = formats.get('default')
187
format.initialize(transport)
188
return format.open(transport)
189
190
@classmethod
191
def from_repository(cls, repository):
192
"""Open a cache file for a repository.
193
194
This will use the repository's transport to store the cache file, or
195
use the users global cache directory if the repository has no
196
transport associated with it.
197
198
:param repository: Repository to open the cache for
199
:return: A `BzrGitCache`
200
"""
201
repo_transport = getattr(repository, "_transport", None)
202
if repo_transport is not None:
203
# Even if we don't write to this repo, we should be able
204
# to update its cache.
205
repo_transport = remove_readonly_transport_decorator(repo_transport)
206
try:
207
repo_transport.mkdir('git')
208
except bzrlib.errors.FileExists:
209
pass
210
transport = repo_transport.clone('git')
211
else:
212
transport = get_remote_cache_transport()
213
return cls.from_transport(transport)
214
215
216
class CacheUpdater(object):
217
"""Base class for objects that can update a bzr-git cache."""
218
219
def add_object(self, obj, ie, path):
220
raise NotImplementedError(self.add_object)
221
222
def finish(self):
223
raise NotImplementedError(self.finish)
224
225
226
class BzrGitCache(object):
227
"""Caching backend."""
228
229
def __init__(self, idmap, content_cache, cache_updater_klass):
230
self.idmap = idmap
231
self.content_cache = content_cache
232
self._cache_updater_klass = cache_updater_klass
233
234
def get_updater(self, rev):
235
"""Update an object that implements the CacheUpdater interface for
236
updating this cache.
237
"""
238
return self._cache_updater_klass(self, rev)
239
240
241
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), None, DictCacheUpdater)
242
243
244
class DictCacheUpdater(CacheUpdater):
245
"""Cache updater for dict-based caches."""
246
247
def __init__(self, cache, rev):
248
self.cache = cache
249
self.revid = rev.revision_id
250
self.parent_revids = rev.parent_ids
251
self._commit = None
252
self._entries = []
253
254
def add_object(self, obj, ie, path):
255
if obj.type_name == "commit":
256
self._commit = obj
257
assert ie is None
258
type_data = (self.revid, self._commit.tree)
259
self.cache.idmap._by_revid[self.revid] = obj.id
260
elif obj.type_name in ("blob", "tree"):
261
if ie is not None:
262
if obj.type_name == "blob":
263
revision = ie.revision
264
else:
265
revision = self.revid
266
type_data = (ie.file_id, revision)
267
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] =\
268
obj.id
269
else:
270
raise AssertionError
271
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
272
273
def finish(self):
274
if self._commit is None:
275
raise AssertionError("No commit object added")
276
return self._commit
277
278
134
279
class DictGitShaMap(GitShaMap):
280
"""Git SHA map that uses a dictionary."""
135
281
136
282
def __init__(self):
137
self.dict = {}
283
self._by_sha = {}
284
self._by_fileid = {}
285
self._by_revid = {}
138
286
139
def add_entry(self, sha, type, type_data):
140
self.dict[sha] = (type, type_data)
287
def lookup_blob_id(self, fileid, revision):
288
return self._by_fileid[revision][fileid]
141
289
142
290
def lookup_git_sha(self, sha):
143
return self.dict[sha]
144
145
def lookup_tree(self, fileid, revid):
146
for k, v in self.dict.iteritems():
147
if v == ("tree", (fileid, revid)):
148
return k
149
raise KeyError((fileid, revid))
150
151
def lookup_blob(self, fileid, revid):
152
for k, v in self.dict.iteritems():
153
if v == ("blob", (fileid, revid)):
154
return k
155
raise KeyError((fileid, revid))
291
return self._by_sha[sha]
292
293
def lookup_tree_id(self, fileid, revision):
294
return self._by_fileid[revision][fileid]
295
296
def lookup_commit(self, revid):
297
return self._by_revid[revid]
156
298
157
299
def revids(self):
158
for key, (type, type_data) in self.dict.iteritems():
300
for key, (type, type_data) in self._by_sha.iteritems():
159
301
if type == "commit":
160
302
yield type_data[0]
161
303
162
304
def sha1s(self):
163
return self.dict.iterkeys()
305
return self._by_sha.iterkeys()
306
307
308
class SqliteCacheUpdater(CacheUpdater):
309
310
def __init__(self, cache, rev):
311
self.cache = cache
312
self.db = self.cache.idmap.db
313
self.revid = rev.revision_id
314
self._commit = None
315
self._trees = []
316
self._blobs = []
317
318
def add_object(self, obj, ie, path):
319
if obj.type_name == "commit":
320
self._commit = obj
321
assert ie is None
322
elif obj.type_name == "tree":
323
if ie is not None:
324
self._trees.append((obj.id, ie.file_id, self.revid))
325
elif obj.type_name == "blob":
326
if ie is not None:
327
self._blobs.append((obj.id, ie.file_id, ie.revision))
328
else:
329
raise AssertionError
330
331
def finish(self):
332
if self._commit is None:
333
raise AssertionError("No commit object added")
334
self.db.executemany(
335
"replace into trees (sha1, fileid, revid) values (?, ?, ?)",
336
self._trees)
337
self.db.executemany(
338
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
339
self._blobs)
340
self.db.execute(
341
"replace into commits (sha1, revid, tree_sha) values (?, ?, ?)",
342
(self._commit.id, self.revid, self._commit.tree))
343
return self._commit
344
345
346
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), None, SqliteCacheUpdater)
347
348
349
class SqliteGitCacheFormat(BzrGitCacheFormat):
350
351
def get_format_string(self):
352
return 'bzr-git sha map version 1 using sqlite\n'
353
354
def open(self, transport):
355
try:
356
basepath = transport.local_abspath(".")
357
except bzrlib.errors.NotLocalUrl:
358
basepath = get_cache_dir()
359
return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))
164
360
165
361
166
362
class SqliteGitShaMap(GitShaMap):
363
"""Bazaar GIT Sha map that uses a sqlite database for storage."""
167
364
168
365
def __init__(self, path=None):
169
366
self.path = path
190
387
create index if not exists blobs_sha1 on blobs(sha1);
191
388
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
192
389
create table if not exists trees(
193
sha1 text not null check(length(sha1) == 40),
390
sha1 text unique not null check(length(sha1) == 40),
194
391
fileid text not null,
195
392
revid text not null
196
393
);
197
create index if not exists trees_sha1 on trees(sha1);
394
create unique index if not exists trees_sha1 on trees(sha1);
198
395
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
199
396
""")
200
397
201
@classmethod
202
def from_repository(cls, repository):
203
try:
204
transport = getattr(repository, "_transport", None)
205
if transport is not None:
206
return cls(os.path.join(transport.local_abspath("."), "git.db"))
207
except bzrlib.errors.NotLocalUrl:
208
pass
209
return cls(os.path.join(get_cache_dir(), "remote.db"))
210
398
def __repr__(self):
399
return "%s(%r)" % (self.__class__.__name__, self.path)
400
211
401
def lookup_commit(self, revid):
212
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
402
cursor = self.db.execute("select sha1 from commits where revid = ?",
403
(revid,))
404
row = cursor.fetchone()
213
405
if row is not None:
214
406
return row[0]
215
407
raise KeyError
217
409
def commit_write_group(self):
218
410
self.db.commit()
219
411
220
def add_entries(self, entries):
221
trees = []
222
blobs = []
223
for sha, type, type_data in entries:
224
assert isinstance(type_data[0], str)
225
assert isinstance(type_data[1], str)
226
entry = (sha, type_data[0], type_data[1])
227
if type == "tree":
228
trees.append(entry)
229
elif type == "blob":
230
blobs.append(entry)
231
else:
232
raise AssertionError
233
if trees:
234
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
235
if blobs:
236
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
237
238
239
def add_entry(self, sha, type, type_data):
240
"""Add a new entry to the database.
241
"""
242
assert isinstance(type_data, tuple)
243
assert isinstance(sha, str), "type was %r" % sha
244
if type == "commit":
245
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
246
elif type in ("blob", "tree"):
247
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
248
else:
249
raise AssertionError("Unknown type %s" % type)
250
251
def lookup_tree(self, fileid, revid):
252
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid,revid)).fetchone()
253
if row is None:
254
raise KeyError((fileid, revid))
255
return row[0]
256
257
def lookup_blob(self, fileid, revid):
258
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
259
if row is None:
260
raise KeyError((fileid, revid))
261
return row[0]
412
def lookup_blob_id(self, fileid, revision):
413
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
414
if row is not None:
415
return row[0]
416
raise KeyError(fileid)
417
418
def lookup_tree_id(self, fileid, revision):
419
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
420
if row is not None:
421
return row[0]
422
raise KeyError(fileid)
262
423
263
424
def lookup_git_sha(self, sha):
264
425
"""Lookup a Git sha in the database.
267
428
:return: (type, type_data) with type_data:
268
429
revision: revid, tree sha
269
430
"""
270
def format(type, row):
271
return (type, (row[0], row[1]))
272
431
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
273
432
if row is not None:
274
return format("commit", row)
433
return ("commit", row)
275
434
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
276
435
if row is not None:
277
return format("blob", row)
436
return ("blob", row)
278
437
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
279
438
if row is not None:
280
return format("tree", row)
439
return ("tree", row)
281
440
raise KeyError(sha)
282
441
283
442
def revids(self):
284
443
"""List the revision ids known."""
285
for row in self.db.execute("select revid from commits").fetchall():
286
yield row[0]
444
return (row for (row,) in self.db.execute("select revid from commits"))
287
445
288
446
def sha1s(self):
289
447
"""List the SHA1s."""
290
448
for table in ("blobs", "commits", "trees"):
291
for row in self.db.execute("select sha1 from %s" % table).fetchall():
292
yield row[0]
293
294
295
TDB_MAP_VERSION = 2
296
TDB_HASH_SIZE = 50000
449
for (sha,) in self.db.execute("select sha1 from %s" % table):
450
yield sha
451
452
453
class TdbCacheUpdater(CacheUpdater):
454
"""Cache updater for tdb-based caches."""
455
456
def __init__(self, cache, rev):
457
self.cache = cache
458
self.db = cache.idmap.db
459
self.revid = rev.revision_id
460
self.parent_revids = rev.parent_ids
461
self._commit = None
462
self._entries = []
463
464
def add_object(self, obj, ie, path):
465
sha = obj.sha().digest()
466
if obj.type_name == "commit":
467
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
468
type_data = (self.revid, obj.tree)
469
self._commit = obj
470
assert ie is None
471
elif obj.type_name == "blob":
472
if ie is None:
473
return
474
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
475
type_data = (ie.file_id, ie.revision)
476
elif obj.type_name == "tree":
477
if ie is None:
478
return
479
type_data = (ie.file_id, self.revid)
480
else:
481
raise AssertionError
482
self.db["git\0" + sha] = "\0".join((obj.type_name, ) + type_data)
483
484
def finish(self):
485
if self._commit is None:
486
raise AssertionError("No commit object added")
487
return self._commit
488
489
490
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
491
492
class TdbGitCacheFormat(BzrGitCacheFormat):
493
"""Cache format for tdb-based caches."""
494
495
def get_format_string(self):
496
return 'bzr-git sha map version 3 using tdb\n'
497
498
def open(self, transport):
499
try:
500
basepath = transport.local_abspath(".")
501
except bzrlib.errors.NotLocalUrl:
502
basepath = get_cache_dir()
503
try:
504
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
505
except ImportError:
506
raise ImportError(
507
"Unable to open existing bzr-git cache because 'tdb' is not "
508
"installed.")
297
509
298
510
299
511
class TdbGitShaMap(GitShaMap):
307
519
"blob fileid revid" -> "<sha1>"
308
520
"""
309
521
522
TDB_MAP_VERSION = 3
523
TDB_HASH_SIZE = 50000
524
310
525
def __init__(self, path=None):
311
526
import tdb
312
527
self.path = path
314
529
self.db = {}
315
530
else:
316
531
if not mapdbs().has_key(path):
317
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
532
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
318
533
os.O_RDWR|os.O_CREAT)
319
534
self.db = mapdbs()[path]
320
535
try:
321
if int(self.db["version"]) != TDB_MAP_VERSION:
536
if int(self.db["version"]) not in (2, 3):
322
537
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
323
self.db["version"], TDB_MAP_VERSION)
538
self.db["version"], self.TDB_MAP_VERSION)
324
539
self.db.clear()
325
self.db["version"] = str(TDB_MAP_VERSION)
326
540
except KeyError:
327
self.db["version"] = str(TDB_MAP_VERSION)
328
329
@classmethod
330
def from_repository(cls, repository):
331
try:
332
transport = getattr(repository, "_transport", None)
333
if transport is not None:
334
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
335
except bzrlib.errors.NotLocalUrl:
336
541
pass
337
return cls(os.path.join(get_cache_dir(), "remote.tdb"))
542
self.db["version"] = str(self.TDB_MAP_VERSION)
543
544
def start_write_group(self):
545
"""Start writing changes."""
546
self.db.transaction_start()
547
548
def commit_write_group(self):
549
"""Commit any pending changes."""
550
self.db.transaction_commit()
551
552
def abort_write_group(self):
553
"""Abort any pending changes."""
554
self.db.transaction_cancel()
555
556
def __repr__(self):
557
return "%s(%r)" % (self.__class__.__name__, self.path)
338
558
339
559
def lookup_commit(self, revid):
340
560
return sha_to_hex(self.db["commit\0" + revid][:20])
341
561
342
def add_entry(self, hexsha, type, type_data):
343
"""Add a new entry to the database.
344
"""
345
if hexsha is None:
346
sha = ""
347
else:
348
sha = hex_to_sha(hexsha)
349
self.db["git\0" + sha] = "\0".join((type, type_data[0], type_data[1]))
350
if type == "commit":
351
self.db["commit\0" + type_data[0]] = "\0".join((sha, type_data[1]))
352
else:
353
self.db["\0".join((type, type_data[0], type_data[1]))] = sha
354
355
def lookup_tree(self, fileid, revid):
356
sha = self.db["\0".join(("tree", fileid, revid))]
357
if sha == "":
358
return None
359
else:
360
return sha_to_hex(sha)
361
362
def lookup_blob(self, fileid, revid):
363
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
364
562
def lookup_blob_id(self, fileid, revision):
563
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
564
365
565
def lookup_git_sha(self, sha):
366
566
"""Lookup a Git sha in the database.
367
567
392
592
for key in self.db.iterkeys():
393
593
if key.startswith("git\0"):
394
594
yield sha_to_hex(key[4:])
595
596
597
class VersionedFilesContentCache(ContentCache):
598
599
def __init__(self, vf):
600
self._vf = vf
601
602
def add(self, obj):
603
self._vf.insert_record_stream(
604
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
605
obj.as_legacy_object_chunks())])
606
607
def __getitem__(self, sha):
608
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
609
entry = stream.next()
610
if entry.storage_kind == 'absent':
611
raise KeyError(sha)
612
return ShaFile._parse_legacy_object(entry.get_bytes_as('fulltext'))
613
614
615
class GitObjectStoreContentCache(ContentCache):
616
617
def __init__(self, store):
618
self.store = store
619
620
def add_multi(self, objs):
621
self.store.add_objects(objs)
622
623
def add(self, obj, path):
624
self.store.add_object(obj)
625
626
def __getitem__(self, sha):
627
return self.store[sha]
628
629
630
class IndexCacheUpdater(CacheUpdater):
631
632
def __init__(self, cache, rev):
633
self.cache = cache
634
self.revid = rev.revision_id
635
self.parent_revids = rev.parent_ids
636
self._commit = None
637
self._entries = []
638
self._cache_objs = set()
639
640
def add_object(self, obj, ie, path):
641
if obj.type_name == "commit":
642
self._commit = obj
643
assert ie is None
644
self.cache.idmap._add_git_sha(obj.id, "commit",
645
(self.revid, obj.tree))
646
self.cache.idmap._add_node(("commit", self.revid, "X"),
647
" ".join((obj.id, obj.tree)))
648
self._cache_objs.add((obj, path))
649
elif obj.type_name == "blob":
650
self.cache.idmap._add_git_sha(obj.id, "blob",
651
(ie.file_id, ie.revision))
652
self.cache.idmap._add_node(("blob", ie.file_id, ie.revision), obj.id)
653
if ie.kind == "symlink":
654
self._cache_objs.add((obj, path))
655
elif obj.type_name == "tree":
656
self.cache.idmap._add_git_sha(obj.id, "tree",
657
(ie.file_id, self.revid))
658
self._cache_objs.add((obj, path))
659
else:
660
raise AssertionError
661
662
def finish(self):
663
self.cache.content_cache.add_multi(self._cache_objs)
664
return self._commit
665
666
667
class IndexBzrGitCache(BzrGitCache):
668
669
def __init__(self, transport=None):
670
mapper = versionedfile.ConstantMapper("trees")
671
shamap = IndexGitShaMap(transport.clone('index'))
672
#trees_store = knit.make_file_factory(True, mapper)(transport)
673
#content_cache = VersionedFilesContentCache(trees_store)
674
from bzrlib.plugins.git.transportgit import TransportObjectStore
675
store = TransportObjectStore(transport.clone('objects'))
676
content_cache = GitObjectStoreContentCache(store)
677
super(IndexBzrGitCache, self).__init__(shamap, content_cache,
678
IndexCacheUpdater)
679
680
681
class IndexGitCacheFormat(BzrGitCacheFormat):
682
683
def get_format_string(self):
684
return 'bzr-git sha map with git object cache version 1\n'
685
686
def initialize(self, transport):
687
super(IndexGitCacheFormat, self).initialize(transport)
688
transport.mkdir('index')
689
transport.mkdir('objects')
690
from bzrlib.plugins.git.transportgit import TransportObjectStore
691
TransportObjectStore.init(transport.clone('objects'))
692
693
def open(self, transport):
694
return IndexBzrGitCache(transport)
695
696
697
class IndexGitShaMap(GitShaMap):
698
"""SHA Map that uses the Bazaar APIs to store a cache.
699
700
BTree Index file with the following contents:
701
702
("git", <sha1>) -> "<type> <type-data1> <type-data2>"
703
("commit", <revid>) -> "<sha1> <tree-id>"
704
("blob", <fileid>, <revid>) -> <sha1>
705
706
"""
707
708
def __init__(self, transport=None):
709
if transport is None:
710
self._transport = None
711
self._index = _mod_index.InMemoryGraphIndex(0, key_elements=3)
712
self._builder = self._index
713
else:
714
self._builder = None
715
self._transport = transport
716
self._index = _mod_index.CombinedGraphIndex([])
717
for name in self._transport.list_dir("."):
718
if not name.endswith(".rix"):
719
continue
720
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
721
self._transport.stat(name).st_size)
722
self._index.insert_index(0, x)
723
724
@classmethod
725
def from_repository(cls, repository):
726
transport = getattr(repository, "_transport", None)
727
if transport is not None:
728
try:
729
transport.mkdir('git')
730
except bzrlib.errors.FileExists:
731
pass
732
return cls(transport.clone('git'))
733
from bzrlib.transport import get_transport
734
return cls(get_transport(get_cache_dir()))
735
736
def __repr__(self):
737
if self._transport is not None:
738
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
739
else:
740
return "%s()" % (self.__class__.__name__)
741
742
def repack(self):
743
assert self._builder is None
744
self.start_write_group()
745
for _, key, value in self._index.iter_all_entries():
746
self._builder.add_node(key, value)
747
to_remove = []
748
for name in self._transport.list_dir('.'):
749
if name.endswith('.rix'):
750
to_remove.append(name)
751
self.commit_write_group()
752
del self._index.indices[1:]
753
for name in to_remove:
754
self._transport.rename(name, name + '.old')
755
756
def start_write_group(self):
757
assert self._builder is None
758
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
759
self._name = osutils.sha()
760
761
def commit_write_group(self):
762
assert self._builder is not None
763
stream = self._builder.finish()
764
name = self._name.hexdigest() + ".rix"
765
size = self._transport.put_file(name, stream)
766
index = _mod_btree_index.BTreeGraphIndex(self._transport, name, size)
767
self._index.insert_index(0, index)
768
self._builder = None
769
self._name = None
770
771
def abort_write_group(self):
772
assert self._builder is not None
773
self._builder = None
774
self._name = None
775
776
def _add_node(self, key, value):
777
try:
778
self._builder.add_node(key, value)
779
except bzrlib.errors.BadIndexDuplicateKey:
780
# Multiple bzr objects can have the same contents
781
return True
782
else:
783
return False
784
785
def _get_entry(self, key):
786
entries = self._index.iter_entries([key])
787
try:
788
return entries.next()[2]
789
except StopIteration:
790
if self._builder is None:
791
raise KeyError
792
entries = self._builder.iter_entries([key])
793
try:
794
return entries.next()[2]
795
except StopIteration:
796
raise KeyError
797
798
def _iter_keys_prefix(self, prefix):
799
for entry in self._index.iter_entries_prefix([prefix]):
800
yield entry[1]
801
if self._builder is not None:
802
for entry in self._builder.iter_entries_prefix([prefix]):
803
yield entry[1]
804
805
def lookup_commit(self, revid):
806
return self._get_entry(("commit", revid, "X"))[:40]
807
808
def _add_git_sha(self, hexsha, type, type_data):
809
if hexsha is not None:
810
self._name.update(hexsha)
811
self._add_node(("git", hexsha, "X"),
812
" ".join((type, type_data[0], type_data[1])))
813
else:
814
# This object is not represented in Git - perhaps an empty
815
# directory?
816
self._name.update(type + " ".join(type_data))
817
818
def lookup_blob_id(self, fileid, revision):
819
return self._get_entry(("blob", fileid, revision))
820
821
def lookup_git_sha(self, sha):
822
if len(sha) == 20:
823
sha = sha_to_hex(sha)
824
data = self._get_entry(("git", sha, "X")).split(" ", 2)
825
return (data[0], (data[1], data[2]))
826
827
def revids(self):
828
"""List the revision ids known."""
829
for key in self._iter_keys_prefix(("commit", None, None)):
830
yield key[1]
831
832
def missing_revisions(self, revids):
833
"""Return set of all the revisions that are not present."""
834
missing_revids = set(revids)
835
for _, key, value in self._index.iter_entries((
836
("commit", revid, "X") for revid in revids)):
837
missing_revids.remove(key[1])
838
return missing_revids
839
840
def sha1s(self):
841
"""List the SHA1s."""
842
for key in self._iter_keys_prefix(("git", None, None)):
843
yield key[1]
844
845
846
formats = registry.Registry()
847
formats.register(TdbGitCacheFormat().get_format_string(),
848
TdbGitCacheFormat())
849
formats.register(SqliteGitCacheFormat().get_format_string(),
850
SqliteGitCacheFormat())
851
formats.register(IndexGitCacheFormat().get_format_string(),
852
IndexGitCacheFormat())
853
# In the future, this will become the default:
854
# formats.register('default', IndexGitCacheFormat())
855
try:
856
import tdb
857
except ImportError:
858
formats.register('default', SqliteGitCacheFormat())
859
else:
860
formats.register('default', TdbGitCacheFormat())
861
862
863
864
def migrate_ancient_formats(repo_transport):
865
# Prefer migrating git.db over git.tdb, since the latter may not
866
# be openable on some platforms.
867
if repo_transport.has("git.db"):
868
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
869
repo_transport.rename("git.db", "git/idmap.db")
870
elif repo_transport.has("git.tdb"):
871
TdbGitCacheFormat().initialize(repo_transport.clone("git"))
872
repo_transport.rename("git.tdb", "git/idmap.tdb")
873
874
875
def remove_readonly_transport_decorator(transport):
876
if transport.is_readonly():
877
return transport._decorated
878
return transport
879
880
881
def from_repository(repository):
882
"""Open a cache file for a repository.
883
884
If the repository is remote and there is no transport available from it
885
this will use a local file in the users cache directory
886
(typically ~/.cache/bazaar/git/)
887
888
:param repository: A repository object
889
"""
890
repo_transport = getattr(repository, "_transport", None)
891
if repo_transport is not None:
892
# Migrate older cache formats
893
repo_transport = remove_readonly_transport_decorator(repo_transport)
894
try:
895
repo_transport.mkdir("git")
896
except bzrlib.errors.FileExists:
897
pass
898
else:
899
migrate_ancient_formats(repo_transport)
900
return BzrGitCacheFormat.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0