34
33
errors as bzr_errors,
40
39
btree_index as _mod_btree_index,
41
40
index as _mod_index,
44
from ..sixish import (
49
from ..transport import (
50
get_transport_from_path,
43
from ...transport import (
54
48
def get_cache_dir():
55
path = os.path.join(bedding.cache_dir(), "git")
56
if not os.path.isdir(path):
50
from xdg.BaseDirectory import xdg_cache_home
52
from ...config import config_dir
53
ret = os.path.join(config_dir(), "git")
55
ret = os.path.join(xdg_cache_home, "breezy", "git")
56
if not os.path.isdir(ret):
61
61
def get_remote_cache_transport(repository):
62
"""Retrieve the transport to use when accessing (unwritable) remote
62
"""Retrieve the transport to use when accessing (unwritable) remote
65
65
uuid = getattr(repository, "uuid", None)
69
69
path = os.path.join(get_cache_dir(), uuid)
70
70
if not os.path.isdir(path):
72
return get_transport_from_path(path)
72
return get_transport(path)
75
75
def check_pysqlite_version(sqlite3):
76
76
"""Check that sqlite library is compatible.
79
if (sqlite3.sqlite_version_info[0] < 3
80
or (sqlite3.sqlite_version_info[0] == 3 and
81
sqlite3.sqlite_version_info[1] < 3)):
79
if (sqlite3.sqlite_version_info[0] < 3 or
80
(sqlite3.sqlite_version_info[0] == 3 and
81
sqlite3.sqlite_version_info[1] < 3)):
82
82
trace.warning('Needs at least sqlite 3.3.x')
83
83
raise bzr_errors.BzrError("incompatible sqlite library")
89
88
check_pysqlite_version(sqlite3)
90
except (ImportError, bzr_errors.BzrError):
89
except (ImportError, bzr_errors.BzrError) as e:
91
90
from pysqlite2 import dbapi2 as sqlite3
92
91
check_pysqlite_version(sqlite3)
94
93
trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
96
95
raise bzr_errors.BzrError("missing sqlite library")
99
98
_mapdbs = threading.local()
103
100
"""Get a cache for this thread's db connections."""
216
213
"""Open a cache file for a repository.
218
215
This will use the repository's transport to store the cache file, or
219
use the users global cache directory if the repository has no
216
use the users global cache directory if the repository has no
220
217
transport associated with it.
222
219
:param repository: Repository to open the cache for
223
220
:return: A `BzrGitCache`
225
from ..transport.local import LocalTransport
222
from ...transport.local import LocalTransport
226
223
repo_transport = getattr(repository, "_transport", None)
227
if (repo_transport is not None
228
and isinstance(repo_transport, LocalTransport)):
224
if (repo_transport is not None and
225
isinstance(repo_transport, LocalTransport)):
229
226
# Even if we don't write to this repo, we should be able
230
227
# to update its cache.
232
repo_transport = remove_readonly_transport_decorator(
229
repo_transport = remove_readonly_transport_decorator(repo_transport)
234
230
except bzr_errors.ReadOnlyError:
271
267
self._cache_updater_klass = cache_updater_klass
273
269
def get_updater(self, rev):
274
"""Update an object that implements the CacheUpdater interface for
270
"""Update an object that implements the CacheUpdater interface for
275
271
updating this cache.
277
273
return self._cache_updater_klass(self, rev)
280
def DictBzrGitCache():
281
return BzrGitCache(DictGitShaMap(), DictCacheUpdater)
276
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), DictCacheUpdater)
284
279
class DictCacheUpdater(CacheUpdater):
295
290
if isinstance(obj, tuple):
296
291
(type_name, hexsha) = obj
298
type_name = obj.type_name.decode('ascii')
293
type_name = obj.type_name
300
if not isinstance(hexsha, bytes):
301
raise TypeError(hexsha)
302
295
if type_name == "commit":
303
296
self._commit = obj
304
297
if type(bzr_key_data) is not dict:
309
302
elif type_name in ("blob", "tree"):
310
303
if bzr_key_data is not None:
311
304
key = type_data = bzr_key_data
312
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[
313
type_data[0]] = hexsha
305
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] = hexsha
315
307
raise AssertionError
316
308
entry = (type_name, type_data)
334
326
return self._by_fileid[revision][fileid]
336
328
def lookup_git_sha(self, sha):
337
if not isinstance(sha, bytes):
339
for entry in viewvalues(self._by_sha[sha]):
329
for entry in self._by_sha[sha].itervalues():
342
332
def lookup_tree_id(self, fileid, revision):
346
336
return self._by_revid[revid]
348
338
def revids(self):
349
for key, entries in viewitems(self._by_sha):
350
for (type, type_data) in viewvalues(entries):
339
for key, entries in self._by_sha.iteritems():
340
for (type, type_data) in entries.values():
351
341
if type == "commit":
352
342
yield type_data[0]
355
return viewkeys(self._by_sha)
345
return self._by_sha.iterkeys()
358
348
class SqliteCacheUpdater(CacheUpdater):
369
359
if isinstance(obj, tuple):
370
360
(type_name, hexsha) = obj
372
type_name = obj.type_name.decode('ascii')
362
type_name = obj.type_name
374
if not isinstance(hexsha, bytes):
375
raise TypeError(hexsha)
376
364
if type_name == "commit":
377
365
self._commit = obj
378
366
if type(bzr_key_data) is not dict:
397
385
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
400
"replace into commits (sha1, revid, tree_sha, testament3_sha1) "
401
"values (?, ?, ?, ?)",
402
(self._commit.id, self.revid, self._commit.tree,
403
self._testament3_sha1))
388
"replace into commits (sha1, revid, tree_sha, testament3_sha1) values (?, ?, ?, ?)",
389
(self._commit.id, self.revid, self._commit.tree, self._testament3_sha1))
404
390
return self._commit
407
def SqliteBzrGitCache(p):
408
return BzrGitCache(SqliteGitShaMap(p), SqliteCacheUpdater)
393
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), SqliteCacheUpdater)
411
396
class SqliteGitCacheFormat(BzrGitCacheFormat):
447
432
revid text not null
449
434
create index if not exists blobs_sha1 on blobs(sha1);
450
create unique index if not exists blobs_fileid_revid on blobs(
435
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
452
436
create table if not exists trees(
453
437
sha1 text unique not null check(length(sha1) == 40),
454
438
fileid text not null,
455
439
revid text not null
457
441
create unique index if not exists trees_sha1 on trees(sha1);
458
create unique index if not exists trees_fileid_revid on trees(
442
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
462
445
self.db.executescript(
463
446
"ALTER TABLE commits ADD testament3_sha1 TEXT;")
464
447
except sqlite3.OperationalError:
465
pass # Column already exists.
448
pass # Column already exists.
467
450
def __repr__(self):
468
451
return "%s(%r)" % (self.__class__.__name__, self.path)
470
453
def lookup_commit(self, revid):
471
cursor = self.db.execute("select sha1 from commits where revid = ?",
454
cursor = self.db.execute("select sha1 from commits where revid = ?",
473
456
row = cursor.fetchone()
474
457
if row is not None:
481
464
def lookup_blob_id(self, fileid, revision):
482
row = self.db.execute(
483
"select sha1 from blobs where fileid = ? and revid = ?",
484
(fileid, revision)).fetchone()
465
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
485
466
if row is not None:
487
468
raise KeyError(fileid)
489
470
def lookup_tree_id(self, fileid, revision):
490
row = self.db.execute(
491
"select sha1 from trees where fileid = ? and revid = ?",
492
(fileid, revision)).fetchone()
471
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
493
472
if row is not None:
495
474
raise KeyError(fileid)
504
483
blob: fileid, revid
507
cursor = self.db.execute(
508
"select revid, tree_sha, testament3_sha1 from commits where "
486
cursor = self.db.execute("select revid, tree_sha, testament3_sha1 from commits where sha1 = ?", (sha,))
510
487
for row in cursor.fetchall():
512
489
if row[2] is not None:
516
493
yield ("commit", (row[0], row[1], verifiers))
517
cursor = self.db.execute(
518
"select fileid, revid from blobs where sha1 = ?", (sha,))
494
cursor = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,))
519
495
for row in cursor.fetchall():
521
497
yield ("blob", row)
522
cursor = self.db.execute(
523
"select fileid, revid from trees where sha1 = ?", (sha,))
498
cursor = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,))
524
499
for row in cursor.fetchall():
526
501
yield ("tree", row)
535
510
"""List the SHA1s."""
536
511
for table in ("blobs", "commits", "trees"):
537
512
for (sha,) in self.db.execute("select sha1 from %s" % table):
538
yield sha.encode('ascii')
541
516
class TdbCacheUpdater(CacheUpdater):
554
529
(type_name, hexsha) = obj
555
530
sha = hex_to_sha(hexsha)
557
type_name = obj.type_name.decode('ascii')
532
type_name = obj.type_name
558
533
sha = obj.sha().digest()
559
534
if type_name == "commit":
560
self.db[b"commit\0" + self.revid] = b"\0".join((sha, obj.tree))
535
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
561
536
if type(bzr_key_data) is not dict:
562
537
raise TypeError(bzr_key_data)
563
538
type_data = (self.revid, obj.tree)
569
544
elif type_name == "blob":
570
545
if bzr_key_data is None:
573
(b"blob", bzr_key_data[0], bzr_key_data[1]))] = sha
547
self.db["\0".join(("blob", bzr_key_data[0], bzr_key_data[1]))] = sha
574
548
type_data = bzr_key_data
575
549
elif type_name == "tree":
576
550
if bzr_key_data is None:
578
552
type_data = bzr_key_data
580
554
raise AssertionError
581
entry = b"\0".join((type_name.encode('ascii'), ) + type_data) + b"\n"
555
entry = "\0".join((type_name, ) + type_data) + "\n"
584
558
oldval = self.db[key]
586
560
self.db[key] = entry
588
if not oldval.endswith(b'\n'):
589
self.db[key] = b"".join([oldval, b"\n", entry])
562
if oldval[-1] != "\n":
563
self.db[key] = "".join([oldval, "\n", entry])
591
self.db[key] = b"".join([oldval, entry])
565
self.db[key] = "".join([oldval, entry])
593
567
def finish(self):
594
568
if self._commit is None:
609
582
def open(self, transport):
611
basepath = transport.local_abspath(".")
584
basepath = transport.local_abspath(".").encode(osutils._fs_enc)
612
585
except bzr_errors.NotLocalUrl:
613
586
basepath = get_cache_dir()
587
if type(basepath) is not str:
588
raise TypeError(basepath)
615
590
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
616
591
except ImportError:
617
if type(path) is not str:
618
raise TypeError(path)
642
619
if path not in mapdbs():
643
620
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
644
os.O_RDWR | os.O_CREAT)
621
os.O_RDWR|os.O_CREAT)
645
622
self.db = mapdbs()[path]
647
if int(self.db[b"version"]) not in (2, 3):
649
"SHA Map is incompatible (%s -> %d), rebuilding database.",
650
self.db[b"version"], self.TDB_MAP_VERSION)
624
if int(self.db["version"]) not in (2, 3):
625
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
626
self.db["version"], self.TDB_MAP_VERSION)
654
self.db[b"version"] = b'%d' % self.TDB_MAP_VERSION
630
self.db["version"] = str(self.TDB_MAP_VERSION)
656
632
def start_write_group(self):
657
633
"""Start writing changes."""
671
647
def lookup_commit(self, revid):
673
return sha_to_hex(self.db[b"commit\0" + revid][:20])
649
return sha_to_hex(self.db["commit\0" + revid][:20])
675
651
raise KeyError("No cache entry for %r" % revid)
677
653
def lookup_blob_id(self, fileid, revision):
678
return sha_to_hex(self.db[b"\0".join((b"blob", fileid, revision))])
654
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
680
656
def lookup_git_sha(self, sha):
681
657
"""Lookup a Git sha in the database.
689
665
if len(sha) == 40:
690
666
sha = hex_to_sha(sha)
691
value = self.db[b"git\0" + sha]
667
value = self.db["git\0" + sha]
692
668
for data in value.splitlines():
693
data = data.split(b"\0")
694
type_name = data[0].decode('ascii')
695
if type_name == "commit":
669
data = data.split("\0")
670
if data[0] == "commit":
696
671
if len(data) == 3:
697
yield (type_name, (data[1], data[2], {}))
672
yield (data[0], (data[1], data[2], {}))
699
yield (type_name, (data[1], data[2],
700
{"testament3-sha1": data[3]}))
701
elif type_name in ("tree", "blob"):
702
yield (type_name, tuple(data[1:]))
674
yield (data[0], (data[1], data[2], {"testament3-sha1": data[3]}))
675
elif data[0] in ("tree", "blob"):
676
yield (data[0], tuple(data[1:]))
704
raise AssertionError("unknown type %r" % type_name)
678
raise AssertionError("unknown type %r" % data[0])
706
680
def missing_revisions(self, revids):
708
682
for revid in revids:
709
if self.db.get(b"commit\0" + revid) is None:
683
if self.db.get("commit\0" + revid) is None:
715
return self.db.keys()
716
except AttributeError: # python < 3
717
return self.db.iterkeys()
719
687
def revids(self):
720
688
"""List the revision ids known."""
721
for key in self._keys():
722
if key.startswith(b"commit\0"):
689
for key in self.db.iterkeys():
690
if key.startswith("commit\0"):
726
694
"""List the SHA1s."""
727
for key in self._keys():
728
if key.startswith(b"git\0"):
695
for key in self.db.iterkeys():
696
if key.startswith("git\0"):
729
697
yield sha_to_hex(key[4:])
737
705
def add(self, obj):
738
706
self._vf.insert_record_stream(
739
[versionedfile.ChunkedContentFactory(
740
(obj.id,), [], None, obj.as_legacy_object_chunks())])
707
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
708
obj.as_legacy_object_chunks())])
742
710
def __getitem__(self, sha):
743
711
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
760
728
if isinstance(obj, tuple):
761
729
(type_name, hexsha) = obj
763
type_name = obj.type_name.decode('ascii')
731
type_name = obj.type_name
765
733
if type_name == "commit":
766
734
self._commit = obj
767
735
if type(bzr_key_data) is not dict:
768
736
raise TypeError(bzr_key_data)
769
self.cache.idmap._add_git_sha(hexsha, b"commit",
770
(self.revid, obj.tree, bzr_key_data))
771
self.cache.idmap._add_node((b"commit", self.revid, b"X"),
772
b" ".join((hexsha, obj.tree)))
737
self.cache.idmap._add_git_sha(hexsha, "commit",
738
(self.revid, obj.tree, bzr_key_data))
739
self.cache.idmap._add_node(("commit", self.revid, "X"),
740
" ".join((hexsha, obj.tree)))
773
741
elif type_name == "blob":
774
self.cache.idmap._add_git_sha(hexsha, b"blob", bzr_key_data)
775
self.cache.idmap._add_node((b"blob", bzr_key_data[0],
776
bzr_key_data[1]), hexsha)
742
self.cache.idmap._add_git_sha(hexsha, "blob", bzr_key_data)
743
self.cache.idmap._add_node(("blob", bzr_key_data[0],
744
bzr_key_data[1]), hexsha)
777
745
elif type_name == "tree":
778
self.cache.idmap._add_git_sha(hexsha, b"tree", bzr_key_data)
746
self.cache.idmap._add_git_sha(hexsha, "tree", bzr_key_data)
780
748
raise AssertionError
786
754
class IndexBzrGitCache(BzrGitCache):
788
756
def __init__(self, transport=None):
757
mapper = versionedfile.ConstantMapper("trees")
789
758
shamap = IndexGitShaMap(transport.clone('index'))
759
from .transportgit import TransportObjectStore
790
760
super(IndexBzrGitCache, self).__init__(shamap, IndexCacheUpdater)
830
800
for name in self._transport.list_dir("."):
831
801
if not name.endswith(".rix"):
833
x = _mod_btree_index.BTreeGraphIndex(
834
self._transport, name, self._transport.stat(name).st_size)
803
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
804
self._transport.stat(name).st_size)
835
805
self._index.insert_index(0, x)
843
813
except bzr_errors.FileExists:
845
815
return cls(transport.clone('git'))
846
return cls(get_transport_from_path(get_cache_dir()))
816
from ...transport import get_transport
817
return cls(get_transport(get_cache_dir()))
848
819
def __repr__(self):
849
820
if self._transport is not None:
854
825
def repack(self):
855
826
if self._builder is not None:
856
raise bzr_errors.BzrError('builder already open')
827
raise errors.BzrError('builder already open')
857
828
self.start_write_group()
858
829
self._builder.add_nodes(
859
830
((key, value) for (_, key, value) in
870
841
def start_write_group(self):
871
842
if self._builder is not None:
872
raise bzr_errors.BzrError('builder already open')
843
raise errors.BzrError('builder already open')
873
844
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
874
845
self._name = osutils.sha()
876
847
def commit_write_group(self):
877
848
if self._builder is None:
878
raise bzr_errors.BzrError('builder not open')
849
raise errors.BzrError('builder not open')
879
850
stream = self._builder.finish()
880
851
name = self._name.hexdigest() + ".rix"
881
852
size = self._transport.put_file(name, stream)
920
891
yield (entry[1], entry[2])
922
893
def lookup_commit(self, revid):
923
return self._get_entry((b"commit", revid, b"X"))[:40]
894
return self._get_entry(("commit", revid, "X"))[:40]
925
896
def _add_git_sha(self, hexsha, type, type_data):
926
897
if hexsha is not None:
927
898
self._name.update(hexsha)
928
if type == b"commit":
929
900
td = (type_data[0], type_data[1])
931
902
td += (type_data[2]["testament3-sha1"],)
936
self._add_node((b"git", hexsha, b"X"), b" ".join((type,) + td))
907
self._add_node(("git", hexsha, "X"), " ".join((type,) + td))
938
909
# This object is not represented in Git - perhaps an empty
940
self._name.update(type + b" ".join(type_data))
911
self._name.update(type + " ".join(type_data))
942
913
def lookup_blob_id(self, fileid, revision):
943
return self._get_entry((b"blob", fileid, revision))
914
return self._get_entry(("blob", fileid, revision))
945
916
def lookup_git_sha(self, sha):
946
917
if len(sha) == 20:
947
918
sha = sha_to_hex(sha)
948
value = self._get_entry((b"git", sha, b"X"))
949
data = value.split(b" ", 3)
950
if data[0] == b"commit":
919
value = self._get_entry(("git", sha, "X"))
920
data = value.split(" ", 3)
921
if data[0] == "commit":
953
924
verifiers = {"testament3-sha1": data[3]}
958
929
yield ("commit", (data[1], data[2], verifiers))
960
yield (data[0].decode('ascii'), tuple(data[1:]))
931
yield (data[0], tuple(data[1:]))
962
933
def revids(self):
963
934
"""List the revision ids known."""
964
for key, value in self._iter_entries_prefix((b"commit", None, None)):
935
for key, value in self._iter_entries_prefix(("commit", None, None)):
967
938
def missing_revisions(self, revids):
968
939
"""Return set of all the revisions that are not present."""
969
940
missing_revids = set(revids)
970
941
for _, key, value in self._index.iter_entries((
971
(b"commit", revid, b"X") for revid in revids)):
942
("commit", revid, "X") for revid in revids)):
972
943
missing_revids.remove(key[1])
973
944
return missing_revids
976
947
"""List the SHA1s."""
977
for key, value in self._iter_entries_prefix((b"git", None, None)):
948
for key, value in self._iter_entries_prefix(("git", None, None)):
981
952
formats = registry.Registry()
982
953
formats.register(TdbGitCacheFormat().get_format_string(),
984
955
formats.register(SqliteGitCacheFormat().get_format_string(),
985
SqliteGitCacheFormat())
956
SqliteGitCacheFormat())
986
957
formats.register(IndexGitCacheFormat().get_format_string(),
987
IndexGitCacheFormat())
958
IndexGitCacheFormat())
988
959
# In the future, this will become the default:
989
960
formats.register('default', IndexGitCacheFormat())
992
964
def migrate_ancient_formats(repo_transport):
993
965
# Migrate older cache formats
994
966
repo_transport = remove_readonly_transport_decorator(repo_transport)
1000
972
repo_transport.mkdir("git")
1001
973
except bzr_errors.FileExists:
1003
# Prefer migrating git.db over git.tdb, since the latter may not
975
# Prefer migrating git.db over git.tdb, since the latter may not
1004
976
# be openable on some platforms.
1006
978
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
1034
1006
migrate_ancient_formats(repo_transport)
1035
1007
except bzr_errors.ReadOnlyError:
1036
pass # Not much we can do
1008
pass # Not much we can do
1037
1009
return BzrGitCacheFormat.from_repository(repository)
added
removed
breezy/plugins/git/cache.py
