To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to shamap.py
- browse files at revision 0.200.866
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.866
- Committer: Jelmer Vernooij
- Date: 2010-04-03 11:13:10 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100403111310-ppw3a7v4n0ogyg5e
More docstrings, prefer migrating git.db to migrating git.tdb.
- files removed:
- help.py
- files renamed:
- cache.py => shamap.py
- tests/test_cache.py => tests/test_shamap.py
- files modified:
- Makefile
added
removed
shamap.py
23
23
import os
24
24
import threading
25
25
26
from dulwich.objects import (
27
ShaFile,
28
)
29
30
26
import bzrlib
31
27
from bzrlib import (
32
btree_index as _mod_btree_index,
33
index as _mod_index,
34
osutils,
35
28
registry,
36
29
trace,
37
versionedfile,
38
30
)
39
31
from bzrlib.transport import (
40
32
get_transport,
55
47
56
48
57
49
def get_remote_cache_transport():
58
"""Retrieve the transport to use when accessing (unwritable) remote
59
repositories.
60
"""
61
50
return get_transport(get_cache_dir())
62
51
63
52
101
90
"""Lookup a Git sha in the database.
102
91
:param sha: Git object sha
103
92
:return: (type, type_data) with type_data:
104
commit: revid, tree_sha, verifiers
105
blob: fileid, revid
106
tree: fileid, revid
93
revision: revid, tree sha
107
94
"""
108
95
raise NotImplementedError(self.lookup_git_sha)
109
96
148
135
class ContentCache(object):
149
136
"""Object that can cache Git objects."""
150
137
151
def add(self, object):
152
"""Add an object."""
153
raise NotImplementedError(self.add)
154
155
def add_multi(self, objects):
156
"""Add multiple objects."""
157
for obj in objects:
158
self.add(obj)
159
160
138
def __getitem__(self, sha):
161
139
"""Retrieve an item, by SHA."""
162
140
raise NotImplementedError(self.__getitem__)
163
141
164
142
165
143
class BzrGitCacheFormat(object):
166
"""Bazaar-Git Cache Format."""
167
144
168
145
def get_format_string(self):
169
146
"""Return a single-line unique format string for this cache format."""
174
151
raise NotImplementedError(self.open)
175
152
176
153
def initialize(self, transport):
177
"""Create a new instance of this cache format at transport."""
178
154
transport.put_bytes('format', self.get_format_string())
179
155
180
156
@classmethod
219
195
220
196
221
197
class CacheUpdater(object):
222
"""Base class for objects that can update a bzr-git cache."""
223
224
def add_object(self, obj, ie, path):
225
"""Add an object.
226
227
:param obj: Object type ("commit", "blob" or "tree")
228
:param ie: Inventory entry (for blob/tree) or testament_sha in case
229
of commit
230
:param path: Path of the object (optional)
231
"""
198
199
def add_object(self, obj, ie):
232
200
raise NotImplementedError(self.add_object)
233
201
234
202
def finish(self):
244
212
self._cache_updater_klass = cache_updater_klass
245
213
246
214
def get_updater(self, rev):
247
"""Update an object that implements the CacheUpdater interface for
248
updating this cache.
249
"""
250
215
return self._cache_updater_klass(self, rev)
251
216
252
217
254
219
255
220
256
221
class DictCacheUpdater(CacheUpdater):
257
"""Cache updater for dict-based caches."""
258
222
259
223
def __init__(self, cache, rev):
260
224
self.cache = cache
263
227
self._commit = None
264
228
self._entries = []
265
229
266
def add_object(self, obj, ie, path):
230
def add_object(self, obj, ie):
267
231
if obj.type_name == "commit":
268
232
self._commit = obj
269
assert type(ie) is dict
270
type_data = (self.revid, self._commit.tree, ie)
233
assert ie is None
234
type_data = (self.revid, self._commit.tree)
271
235
self.cache.idmap._by_revid[self.revid] = obj.id
272
236
elif obj.type_name in ("blob", "tree"):
273
if ie is not None:
274
if obj.type_name == "blob":
275
revision = ie.revision
276
else:
277
revision = self.revid
278
type_data = (ie.file_id, revision)
279
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] = obj.id
237
if obj.type_name == "blob":
238
revision = ie.revision
239
else:
240
revision = self.revid
241
type_data = (ie.file_id, revision)
242
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] = obj.id
280
243
else:
281
244
raise AssertionError
282
245
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
288
251
289
252
290
253
class DictGitShaMap(GitShaMap):
291
"""Git SHA map that uses a dictionary."""
292
254
293
255
def __init__(self):
294
256
self._by_sha = {}
326
288
self._trees = []
327
289
self._blobs = []
328
290
329
def add_object(self, obj, ie, path):
291
def add_object(self, obj, ie):
330
292
if obj.type_name == "commit":
331
293
self._commit = obj
332
self._testament3_sha1 = ie["testament3-sha1"]
333
assert type(ie) is dict
294
assert ie is None
334
295
elif obj.type_name == "tree":
335
if ie is not None:
336
self._trees.append((obj.id, ie.file_id, self.revid))
296
self._trees.append((obj.id, ie.file_id, self.revid))
337
297
elif obj.type_name == "blob":
338
if ie is not None:
339
self._blobs.append((obj.id, ie.file_id, ie.revision))
298
self._blobs.append((obj.id, ie.file_id, ie.revision))
340
299
else:
341
300
raise AssertionError
342
301
350
309
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
351
310
self._blobs)
352
311
self.db.execute(
353
"replace into commits (sha1, revid, tree_sha, testament3_sha1) values (?, ?, ?, ?)",
354
(self._commit.id, self.revid, self._commit.tree, self._testament3_sha1))
312
"replace into commits (sha1, revid, tree_sha) values (?, ?, ?)",
313
(self._commit.id, self.revid, self._commit.tree))
355
314
return self._commit
356
315
357
316
372
331
373
332
374
333
class SqliteGitShaMap(GitShaMap):
375
"""Bazaar GIT Sha map that uses a sqlite database for storage."""
376
334
377
335
def __init__(self, path=None):
378
336
self.path = path
406
364
create unique index if not exists trees_sha1 on trees(sha1);
407
365
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
408
366
""")
409
try:
410
self.db.executescript(
411
"ALTER TABLE commits ADD testament3_sha1 TEXT;")
412
except sqlite3.OperationalError:
413
pass # Column already exists.
414
367
415
368
def __repr__(self):
416
369
return "%s(%r)" % (self.__class__.__name__, self.path)
417
370
418
371
def lookup_commit(self, revid):
419
cursor = self.db.execute("select sha1 from commits where revid = ?",
420
(revid,))
421
row = cursor.fetchone()
372
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
422
373
if row is not None:
423
374
return row[0]
424
375
raise KeyError
443
394
444
395
:param sha: Git object sha
445
396
:return: (type, type_data) with type_data:
446
commit: revid, tree sha, verifiers
447
tree: fileid, revid
448
blob: fileid, revid
397
revision: revid, tree sha
449
398
"""
450
row = self.db.execute("select revid, tree_sha, testament3_sha1 from commits where sha1 = ?", (sha,)).fetchone()
399
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
451
400
if row is not None:
452
return ("commit", (row[0], row[1], {"testament3-sha1": row[2]}))
401
return ("commit", row)
453
402
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
454
403
if row is not None:
455
404
return ("blob", row)
470
419
471
420
472
421
class TdbCacheUpdater(CacheUpdater):
473
"""Cache updater for tdb-based caches."""
474
422
475
423
def __init__(self, cache, rev):
476
424
self.cache = cache
480
428
self._commit = None
481
429
self._entries = []
482
430
483
def add_object(self, obj, ie, path):
431
def add_object(self, obj, ie):
484
432
sha = obj.sha().digest()
485
433
if obj.type_name == "commit":
486
434
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
487
assert type(ie) is dict, "was %r" % ie
488
type_data = (self.revid, obj.tree, ie["testament3-sha1"])
435
type_data = (self.revid, obj.tree)
489
436
self._commit = obj
437
assert ie is None
490
438
elif obj.type_name == "blob":
491
if ie is None:
492
return
493
439
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
494
440
type_data = (ie.file_id, ie.revision)
495
441
elif obj.type_name == "tree":
496
if ie is None:
497
return
498
442
type_data = (ie.file_id, self.revid)
499
443
else:
500
444
raise AssertionError
509
453
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
510
454
511
455
class TdbGitCacheFormat(BzrGitCacheFormat):
512
"""Cache format for tdb-based caches."""
513
456
514
457
def get_format_string(self):
515
458
return 'bzr-git sha map version 3 using tdb\n'
580
523
581
524
def lookup_blob_id(self, fileid, revision):
582
525
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
583
526
584
527
def lookup_git_sha(self, sha):
585
528
"""Lookup a Git sha in the database.
586
529
587
530
:param sha: Git object sha
588
531
:return: (type, type_data) with type_data:
589
commit: revid, tree sha
590
blob: fileid, revid
591
tree: fileid, revid
532
revision: revid, tree sha
592
533
"""
593
534
if len(sha) == 40:
594
535
sha = hex_to_sha(sha)
595
536
data = self.db["git\0" + sha].split("\0")
596
if data[0] == "commit":
597
if len(data) == 3:
598
return (data[0], (data[1], data[2], {}))
599
else:
600
return (data[0], (data[1], data[2], {"testament3-sha1": data[3]}))
601
else:
602
return (data[0], tuple(data[1:]))
537
return (data[0], (data[1], data[2]))
603
538
604
539
def missing_revisions(self, revids):
605
540
ret = set()
621
556
yield sha_to_hex(key[4:])
622
557
623
558
624
class VersionedFilesContentCache(ContentCache):
625
626
def __init__(self, vf):
627
self._vf = vf
628
629
def add(self, obj):
630
self._vf.insert_record_stream(
631
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
632
obj.as_legacy_object_chunks())])
633
634
def __getitem__(self, sha):
635
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
636
entry = stream.next()
637
if entry.storage_kind == 'absent':
638
raise KeyError(sha)
639
return ShaFile._parse_legacy_object(entry.get_bytes_as('fulltext'))
640
641
642
class GitObjectStoreContentCache(ContentCache):
643
644
def __init__(self, store):
645
self.store = store
646
647
def add_multi(self, objs):
648
self.store.add_objects(objs)
649
650
def add(self, obj, path):
651
self.store.add_object(obj)
652
653
def __getitem__(self, sha):
654
return self.store[sha]
655
656
657
class IndexCacheUpdater(CacheUpdater):
658
659
def __init__(self, cache, rev):
660
self.cache = cache
661
self.revid = rev.revision_id
662
self.parent_revids = rev.parent_ids
663
self._commit = None
664
self._entries = []
665
self._cache_objs = set()
666
667
def add_object(self, obj, ie, path):
668
if obj.type_name == "commit":
669
self._commit = obj
670
assert type(ie) is dict
671
self.cache.idmap._add_git_sha(obj.id, "commit",
672
(self.revid, obj.tree, ie))
673
self.cache.idmap._add_node(("commit", self.revid, "X"),
674
" ".join((obj.id, obj.tree)))
675
self._cache_objs.add((obj, path))
676
elif obj.type_name == "blob":
677
self.cache.idmap._add_git_sha(obj.id, "blob",
678
(ie.file_id, ie.revision))
679
self.cache.idmap._add_node(("blob", ie.file_id, ie.revision), obj.id)
680
if ie.kind == "symlink":
681
self._cache_objs.add((obj, path))
682
elif obj.type_name == "tree":
683
self.cache.idmap._add_git_sha(obj.id, "tree",
684
(ie.file_id, self.revid))
685
self._cache_objs.add((obj, path))
686
else:
687
raise AssertionError
688
689
def finish(self):
690
self.cache.content_cache.add_multi(self._cache_objs)
691
return self._commit
692
693
694
class IndexBzrGitCache(BzrGitCache):
695
696
def __init__(self, transport=None):
697
mapper = versionedfile.ConstantMapper("trees")
698
shamap = IndexGitShaMap(transport.clone('index'))
699
#trees_store = knit.make_file_factory(True, mapper)(transport)
700
#content_cache = VersionedFilesContentCache(trees_store)
701
from bzrlib.plugins.git.transportgit import TransportObjectStore
702
store = TransportObjectStore(transport.clone('objects'))
703
content_cache = GitObjectStoreContentCache(store)
704
super(IndexBzrGitCache, self).__init__(shamap, content_cache,
705
IndexCacheUpdater)
706
707
708
class IndexGitCacheFormat(BzrGitCacheFormat):
709
710
def get_format_string(self):
711
return 'bzr-git sha map with git object cache version 1\n'
712
713
def initialize(self, transport):
714
super(IndexGitCacheFormat, self).initialize(transport)
715
transport.mkdir('index')
716
transport.mkdir('objects')
717
from bzrlib.plugins.git.transportgit import TransportObjectStore
718
TransportObjectStore.init(transport.clone('objects'))
719
720
def open(self, transport):
721
return IndexBzrGitCache(transport)
722
723
724
class IndexGitShaMap(GitShaMap):
725
"""SHA Map that uses the Bazaar APIs to store a cache.
726
727
BTree Index file with the following contents:
728
729
("git", <sha1>) -> "<type> <type-data1> <type-data2>"
730
("commit", <revid>) -> "<sha1> <tree-id>"
731
("blob", <fileid>, <revid>) -> <sha1>
732
733
"""
734
735
def __init__(self, transport=None):
736
if transport is None:
737
self._transport = None
738
self._index = _mod_index.InMemoryGraphIndex(0, key_elements=3)
739
self._builder = self._index
740
else:
741
self._builder = None
742
self._transport = transport
743
self._index = _mod_index.CombinedGraphIndex([])
744
for name in self._transport.list_dir("."):
745
if not name.endswith(".rix"):
746
continue
747
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
748
self._transport.stat(name).st_size)
749
self._index.insert_index(0, x)
750
751
@classmethod
752
def from_repository(cls, repository):
753
transport = getattr(repository, "_transport", None)
754
if transport is not None:
755
try:
756
transport.mkdir('git')
757
except bzrlib.errors.FileExists:
758
pass
759
return cls(transport.clone('git'))
760
from bzrlib.transport import get_transport
761
return cls(get_transport(get_cache_dir()))
762
763
def __repr__(self):
764
if self._transport is not None:
765
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
766
else:
767
return "%s()" % (self.__class__.__name__)
768
769
def repack(self):
770
assert self._builder is None
771
self.start_write_group()
772
for _, key, value in self._index.iter_all_entries():
773
self._builder.add_node(key, value)
774
to_remove = []
775
for name in self._transport.list_dir('.'):
776
if name.endswith('.rix'):
777
to_remove.append(name)
778
self.commit_write_group()
779
del self._index.indices[1:]
780
for name in to_remove:
781
self._transport.rename(name, name + '.old')
782
783
def start_write_group(self):
784
assert self._builder is None
785
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
786
self._name = osutils.sha()
787
788
def commit_write_group(self):
789
assert self._builder is not None
790
stream = self._builder.finish()
791
name = self._name.hexdigest() + ".rix"
792
size = self._transport.put_file(name, stream)
793
index = _mod_btree_index.BTreeGraphIndex(self._transport, name, size)
794
self._index.insert_index(0, index)
795
self._builder = None
796
self._name = None
797
798
def abort_write_group(self):
799
assert self._builder is not None
800
self._builder = None
801
self._name = None
802
803
def _add_node(self, key, value):
804
try:
805
self._builder.add_node(key, value)
806
except bzrlib.errors.BadIndexDuplicateKey:
807
# Multiple bzr objects can have the same contents
808
return True
809
else:
810
return False
811
812
def _get_entry(self, key):
813
entries = self._index.iter_entries([key])
814
try:
815
return entries.next()[2]
816
except StopIteration:
817
if self._builder is None:
818
raise KeyError
819
entries = self._builder.iter_entries([key])
820
try:
821
return entries.next()[2]
822
except StopIteration:
823
raise KeyError
824
825
def _iter_keys_prefix(self, prefix):
826
for entry in self._index.iter_entries_prefix([prefix]):
827
yield entry[1]
828
if self._builder is not None:
829
for entry in self._builder.iter_entries_prefix([prefix]):
830
yield entry[1]
831
832
def lookup_commit(self, revid):
833
return self._get_entry(("commit", revid, "X"))[:40]
834
835
def _add_git_sha(self, hexsha, type, type_data):
836
if hexsha is not None:
837
self._name.update(hexsha)
838
if type == "commit":
839
td = (type_data[0], type_data[1], type_data[2]["testament3-sha1"])
840
else:
841
td = type_data
842
self._add_node(("git", hexsha, "X"), " ".join((type,) + td))
843
else:
844
# This object is not represented in Git - perhaps an empty
845
# directory?
846
self._name.update(type + " ".join(type_data))
847
848
def lookup_blob_id(self, fileid, revision):
849
return self._get_entry(("blob", fileid, revision))
850
851
def lookup_git_sha(self, sha):
852
if len(sha) == 20:
853
sha = sha_to_hex(sha)
854
data = self._get_entry(("git", sha, "X")).split(" ", 3)
855
if data[0] == "commit":
856
return ("commit", (data[1], data[2], {"testament3-sha1": data[3]}))
857
else:
858
return (data[0], tuple(data[1:]))
859
860
def revids(self):
861
"""List the revision ids known."""
862
for key in self._iter_keys_prefix(("commit", None, None)):
863
yield key[1]
864
865
def missing_revisions(self, revids):
866
"""Return set of all the revisions that are not present."""
867
missing_revids = set(revids)
868
for _, key, value in self._index.iter_entries((
869
("commit", revid, "X") for revid in revids)):
870
missing_revids.remove(key[1])
871
return missing_revids
872
873
def sha1s(self):
874
"""List the SHA1s."""
875
for key in self._iter_keys_prefix(("git", None, None)):
876
yield key[1]
877
878
879
559
formats = registry.Registry()
880
560
formats.register(TdbGitCacheFormat().get_format_string(),
881
561
TdbGitCacheFormat())
882
562
formats.register(SqliteGitCacheFormat().get_format_string(),
883
563
SqliteGitCacheFormat())
884
formats.register(IndexGitCacheFormat().get_format_string(),
885
IndexGitCacheFormat())
886
# In the future, this will become the default:
887
# formats.register('default', IndexGitCacheFormat())
888
564
try:
889
565
import tdb
890
566
except ImportError:
893
569
formats.register('default', TdbGitCacheFormat())
894
570
895
571
896
897
572
def migrate_ancient_formats(repo_transport):
898
573
# Prefer migrating git.db over git.tdb, since the latter may not
899
574
# be openable on some platforms.
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0