To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to shamap.py
- browse files at revision 0.235.1
- compare with another revision
- download diff
- download tarball
- view history from revision 0.235.1
- Committer: Jelmer Vernooij
- Date: 2009-05-16 21:13:11 UTC
- mto: (0.200.527 trunk)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20090516211311-clcz6rvpnz82cgi4
Store sha map more efficiently.
- files removed:
- help.py
- files renamed:
- cache.py => shamap.py
- tests/test_cache.py => tests/test_shamap.py
- files modified:
- HACKING
added
removed
shamap.py
23
23
import os
24
24
import threading
25
25
26
from dulwich.objects import (
27
ShaFile,
28
)
29
30
26
import bzrlib
31
from bzrlib import (
32
btree_index as _mod_btree_index,
33
index as _mod_index,
34
osutils,
35
registry,
36
trace,
37
versionedfile,
38
)
39
from bzrlib.transport import (
40
get_transport,
41
)
42
43
44
def get_cache_dir():
45
try:
46
from xdg.BaseDirectory import xdg_cache_home
47
except ImportError:
48
from bzrlib.config import config_dir
49
ret = os.path.join(config_dir(), "git")
50
else:
51
ret = os.path.join(xdg_cache_home, "bazaar", "git")
52
if not os.path.isdir(ret):
53
os.makedirs(ret)
54
return ret
55
56
57
def get_remote_cache_transport():
58
"""Retrieve the transport to use when accessing (unwritable) remote
59
repositories.
60
"""
61
return get_transport(get_cache_dir())
27
from bzrlib.errors import (
28
NoSuchRevision,
29
)
62
30
63
31
64
32
def check_pysqlite_version(sqlite3):
65
33
"""Check that sqlite library is compatible.
66
34
67
35
"""
68
if (sqlite3.sqlite_version_info[0] < 3 or
69
(sqlite3.sqlite_version_info[0] == 3 and
36
if (sqlite3.sqlite_version_info[0] < 3 or
37
(sqlite3.sqlite_version_info[0] == 3 and
70
38
sqlite3.sqlite_version_info[1] < 3)):
71
trace.warning('Needs at least sqlite 3.3.x')
39
warning('Needs at least sqlite 3.3.x')
72
40
raise bzrlib.errors.BzrError("incompatible sqlite library")
73
41
74
42
try:
75
43
try:
76
44
import sqlite3
77
45
check_pysqlite_version(sqlite3)
78
except (ImportError, bzrlib.errors.BzrError), e:
46
except (ImportError, bzrlib.errors.BzrError), e:
79
47
from pysqlite2 import dbapi2 as sqlite3
80
48
check_pysqlite_version(sqlite3)
81
49
except:
82
trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
50
warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
83
51
'module')
84
52
raise bzrlib.errors.BzrError("missing sqlite library")
85
53
97
65
class GitShaMap(object):
98
66
"""Git<->Bzr revision id mapping database."""
99
67
68
def add_entry(self, sha, type, type_data):
69
"""Add a new entry to the database.
70
"""
71
raise NotImplementedError(self.add_entry)
72
73
def add_entries(self, entries):
74
"""Add multiple new entries to the database.
75
"""
76
for e in entries:
77
self.add_entry(*e)
78
79
def lookup_tree(self, fileid, revid):
80
"""Lookup the SHA of a git tree."""
81
raise NotImplementedError(self.lookup_tree)
82
83
def lookup_blob(self, fileid, revid):
84
"""Lookup a blob by the fileid it has in a bzr revision."""
85
raise NotImplementedError(self.lookup_blob)
86
100
87
def lookup_git_sha(self, sha):
101
88
"""Lookup a Git sha in the database.
89
102
90
:param sha: Git object sha
103
91
:return: (type, type_data) with type_data:
104
commit: revid, tree_sha, verifiers
105
blob: fileid, revid
106
tree: fileid, revid
92
revision: revid, tree sha
107
93
"""
108
94
raise NotImplementedError(self.lookup_git_sha)
109
95
110
def lookup_blob_id(self, file_id, revision):
111
"""Retrieve a Git blob SHA by file id.
112
113
:param file_id: File id of the file/symlink
114
:param revision: revision in which the file was last changed.
115
"""
116
raise NotImplementedError(self.lookup_blob_id)
117
118
def lookup_tree_id(self, file_id, revision):
119
"""Retrieve a Git tree SHA by file id.
120
"""
121
raise NotImplementedError(self.lookup_tree_id)
122
123
96
def revids(self):
124
97
"""List the revision ids known."""
125
98
raise NotImplementedError(self.revids)
126
99
127
def missing_revisions(self, revids):
128
"""Return set of all the revisions that are not present."""
129
present_revids = set(self.revids())
130
if not isinstance(revids, set):
131
revids = set(revids)
132
return revids - present_revids
133
134
def sha1s(self):
100
def sha1s(Self):
135
101
"""List the SHA1s."""
136
102
raise NotImplementedError(self.sha1s)
137
103
138
def start_write_group(self):
139
"""Start writing changes."""
140
141
def commit_write_group(self):
104
def commit(self):
142
105
"""Commit any pending changes."""
143
106
144
def abort_write_group(self):
145
"""Abort any pending changes."""
146
147
148
class ContentCache(object):
149
"""Object that can cache Git objects."""
150
151
def add(self, object):
152
"""Add an object."""
153
raise NotImplementedError(self.add)
154
155
def add_multi(self, objects):
156
"""Add multiple objects."""
157
for obj in objects:
158
self.add(obj)
159
160
def __getitem__(self, sha):
161
"""Retrieve an item, by SHA."""
162
raise NotImplementedError(self.__getitem__)
163
164
165
class BzrGitCacheFormat(object):
166
"""Bazaar-Git Cache Format."""
167
168
def get_format_string(self):
169
"""Return a single-line unique format string for this cache format."""
170
raise NotImplementedError(self.get_format_string)
171
172
def open(self, transport):
173
"""Open this format on a transport."""
174
raise NotImplementedError(self.open)
175
176
def initialize(self, transport):
177
"""Create a new instance of this cache format at transport."""
178
transport.put_bytes('format', self.get_format_string())
179
180
@classmethod
181
def from_transport(self, transport):
182
"""Open a cache file present on a transport, or initialize one.
183
184
:param transport: Transport to use
185
:return: A BzrGitCache instance
186
"""
187
try:
188
format_name = transport.get_bytes('format')
189
format = formats.get(format_name)
190
except bzrlib.errors.NoSuchFile:
191
format = formats.get('default')
192
format.initialize(transport)
193
return format.open(transport)
194
195
@classmethod
196
def from_repository(cls, repository):
197
"""Open a cache file for a repository.
198
199
This will use the repository's transport to store the cache file, or
200
use the users global cache directory if the repository has no
201
transport associated with it.
202
203
:param repository: Repository to open the cache for
204
:return: A `BzrGitCache`
205
"""
206
repo_transport = getattr(repository, "_transport", None)
207
if repo_transport is not None:
208
# Even if we don't write to this repo, we should be able
209
# to update its cache.
210
repo_transport = remove_readonly_transport_decorator(repo_transport)
211
try:
212
repo_transport.mkdir('git')
213
except bzrlib.errors.FileExists:
214
pass
215
transport = repo_transport.clone('git')
216
else:
217
transport = get_remote_cache_transport()
218
return cls.from_transport(transport)
219
220
221
class CacheUpdater(object):
222
"""Base class for objects that can update a bzr-git cache."""
223
224
def add_object(self, obj, ie, path):
225
"""Add an object.
226
227
:param obj: Object type ("commit", "blob" or "tree")
228
:param ie: Inventory entry (for blob/tree) or testament_sha in case
229
of commit
230
:param path: Path of the object (optional)
231
"""
232
raise NotImplementedError(self.add_object)
233
234
def finish(self):
235
raise NotImplementedError(self.finish)
236
237
238
class BzrGitCache(object):
239
"""Caching backend."""
240
241
def __init__(self, idmap, content_cache, cache_updater_klass):
242
self.idmap = idmap
243
self.content_cache = content_cache
244
self._cache_updater_klass = cache_updater_klass
245
246
def get_updater(self, rev):
247
"""Update an object that implements the CacheUpdater interface for
248
updating this cache.
249
"""
250
return self._cache_updater_klass(self, rev)
251
252
253
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), None, DictCacheUpdater)
254
255
256
class DictCacheUpdater(CacheUpdater):
257
"""Cache updater for dict-based caches."""
258
259
def __init__(self, cache, rev):
260
self.cache = cache
261
self.revid = rev.revision_id
262
self.parent_revids = rev.parent_ids
263
self._commit = None
264
self._entries = []
265
266
def add_object(self, obj, ie, path):
267
if obj.type_name == "commit":
268
self._commit = obj
269
assert type(ie) is dict
270
type_data = (self.revid, self._commit.tree, ie)
271
self.cache.idmap._by_revid[self.revid] = obj.id
272
elif obj.type_name in ("blob", "tree"):
273
if ie is not None:
274
if obj.type_name == "blob":
275
revision = ie.revision
276
else:
277
revision = self.revid
278
type_data = (ie.file_id, revision)
279
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] = obj.id
280
else:
281
raise AssertionError
282
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
283
284
def finish(self):
285
if self._commit is None:
286
raise AssertionError("No commit object added")
287
return self._commit
288
289
107
290
108
class DictGitShaMap(GitShaMap):
291
"""Git SHA map that uses a dictionary."""
292
109
293
110
def __init__(self):
294
self._by_sha = {}
295
self._by_fileid = {}
296
self._by_revid = {}
111
self.dict = {}
297
112
298
def lookup_blob_id(self, fileid, revision):
299
return self._by_fileid[revision][fileid]
113
def add_entry(self, sha, type, type_data):
114
self.dict[sha] = (type, type_data)
300
115
301
116
def lookup_git_sha(self, sha):
302
return self._by_sha[sha]
303
304
def lookup_tree_id(self, fileid, revision):
305
return self._by_fileid[revision][fileid]
306
307
def lookup_commit(self, revid):
308
return self._by_revid[revid]
117
return self.dict[sha]
118
119
def lookup_tree(self, fileid, revid):
120
for k, v in self.dict.iteritems():
121
if v == ("tree", (fileid, revid)):
122
return k
123
raise KeyError((fileid, revid))
124
125
def lookup_blob(self, fileid, revid):
126
for k, v in self.dict.iteritems():
127
if v == ("blob", (fileid, revid)):
128
return k
129
raise KeyError((fileid, revid))
309
130
310
131
def revids(self):
311
for key, (type, type_data) in self._by_sha.iteritems():
132
for key, (type, type_data) in self.dict.iteritems():
312
133
if type == "commit":
313
134
yield type_data[0]
314
135
315
136
def sha1s(self):
316
return self._by_sha.iterkeys()
317
318
319
class SqliteCacheUpdater(CacheUpdater):
320
321
def __init__(self, cache, rev):
322
self.cache = cache
323
self.db = self.cache.idmap.db
324
self.revid = rev.revision_id
325
self._commit = None
326
self._trees = []
327
self._blobs = []
328
329
def add_object(self, obj, ie, path):
330
if obj.type_name == "commit":
331
self._commit = obj
332
self._testament3_sha1 = ie["testament3-sha1"]
333
assert type(ie) is dict
334
elif obj.type_name == "tree":
335
if ie is not None:
336
self._trees.append((obj.id, ie.file_id, self.revid))
337
elif obj.type_name == "blob":
338
if ie is not None:
339
self._blobs.append((obj.id, ie.file_id, ie.revision))
340
else:
341
raise AssertionError
342
343
def finish(self):
344
if self._commit is None:
345
raise AssertionError("No commit object added")
346
self.db.executemany(
347
"replace into trees (sha1, fileid, revid) values (?, ?, ?)",
348
self._trees)
349
self.db.executemany(
350
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
351
self._blobs)
352
self.db.execute(
353
"replace into commits (sha1, revid, tree_sha, testament3_sha1) values (?, ?, ?, ?)",
354
(self._commit.id, self.revid, self._commit.tree, self._testament3_sha1))
355
return self._commit
356
357
358
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), None, SqliteCacheUpdater)
359
360
361
class SqliteGitCacheFormat(BzrGitCacheFormat):
362
363
def get_format_string(self):
364
return 'bzr-git sha map version 1 using sqlite\n'
365
366
def open(self, transport):
367
try:
368
basepath = transport.local_abspath(".")
369
except bzrlib.errors.NotLocalUrl:
370
basepath = get_cache_dir()
371
return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))
137
return self.dict.iterkeys()
372
138
373
139
374
140
class SqliteGitShaMap(GitShaMap):
375
"""Bazaar GIT Sha map that uses a sqlite database for storage."""
376
141
377
142
def __init__(self, path=None):
378
143
self.path = path
381
146
else:
382
147
if not mapdbs().has_key(path):
383
148
mapdbs()[path] = sqlite3.connect(path)
384
self.db = mapdbs()[path]
385
self.db.text_factory = str
149
self.db = mapdbs()[path]
386
150
self.db.executescript("""
387
create table if not exists commits(
388
sha1 text not null check(length(sha1) == 40),
389
revid text not null,
390
tree_sha text not null check(length(tree_sha) == 40)
391
);
151
create table if not exists commits(sha1 text, revid text, tree_sha text);
392
152
create index if not exists commit_sha1 on commits(sha1);
393
153
create unique index if not exists commit_revid on commits(revid);
394
create table if not exists blobs(
395
sha1 text not null check(length(sha1) == 40),
396
fileid text not null,
397
revid text not null
398
);
154
create table if not exists blobs(sha1 text, fileid text, revid text);
399
155
create index if not exists blobs_sha1 on blobs(sha1);
400
156
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
401
create table if not exists trees(
402
sha1 text unique not null check(length(sha1) == 40),
403
fileid text not null,
404
revid text not null
405
);
406
create unique index if not exists trees_sha1 on trees(sha1);
157
create table if not exists trees(sha1 text, fileid text, revid text);
158
create index if not exists trees_sha1 on trees(sha1);
407
159
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
408
160
""")
409
try:
410
self.db.executescript(
411
"ALTER TABLE commits ADD testament3_sha1 TEXT;")
412
except sqlite3.OperationalError:
413
pass # Column already exists.
414
161
415
def __repr__(self):
416
return "%s(%r)" % (self.__class__.__name__, self.path)
162
@classmethod
163
def from_repository(cls, repository):
164
return cls(os.path.join(repository._transport.local_abspath("."), "git.db"))
417
165
418
166
def lookup_commit(self, revid):
419
cursor = self.db.execute("select sha1 from commits where revid = ?",
420
(revid,))
421
row = cursor.fetchone()
167
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
422
168
if row is not None:
423
return row[0]
169
return row[0].encode("utf-8")
424
170
raise KeyError
425
171
426
def commit_write_group(self):
172
def commit(self):
427
173
self.db.commit()
428
174
429
def lookup_blob_id(self, fileid, revision):
430
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
431
if row is not None:
432
return row[0]
433
raise KeyError(fileid)
434
435
def lookup_tree_id(self, fileid, revision):
436
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
437
if row is not None:
438
return row[0]
439
raise KeyError(fileid)
175
def add_entries(self, entries):
176
trees = []
177
blobs = []
178
for sha, type, type_data in entries:
179
assert isinstance(type_data[0], str)
180
assert isinstance(type_data[1], str)
181
entry = (sha.decode("utf-8"), type_data[0].decode("utf-8"),
182
type_data[1].decode("utf-8"))
183
if type == "tree":
184
trees.append(entry)
185
elif type == "blob":
186
blobs.append(entry)
187
else:
188
raise AssertionError
189
if trees:
190
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
191
if blobs:
192
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
193
194
195
def add_entry(self, sha, type, type_data):
196
"""Add a new entry to the database.
197
"""
198
assert isinstance(type_data, tuple)
199
assert isinstance(sha, str), "type was %r" % sha
200
if type == "commit":
201
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
202
elif type in ("blob", "tree"):
203
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
204
else:
205
raise AssertionError("Unknown type %s" % type)
206
207
def lookup_tree(self, fileid, revid):
208
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid,revid)).fetchone()
209
if row is None:
210
raise KeyError((fileid, revid))
211
return row[0].encode("utf-8")
212
213
def lookup_blob(self, fileid, revid):
214
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
215
if row is None:
216
raise KeyError((fileid, revid))
217
return row[0].encode("utf-8")
440
218
441
219
def lookup_git_sha(self, sha):
442
220
"""Lookup a Git sha in the database.
443
221
444
222
:param sha: Git object sha
445
223
:return: (type, type_data) with type_data:
446
commit: revid, tree sha, verifiers
447
tree: fileid, revid
448
blob: fileid, revid
224
revision: revid, tree sha
449
225
"""
450
row = self.db.execute("select revid, tree_sha, testament3_sha1 from commits where sha1 = ?", (sha,)).fetchone()
226
def format(type, row):
227
return (type, (row[0].encode("utf-8"), row[1].encode("utf-8")))
228
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
451
229
if row is not None:
452
return ("commit", (row[0], row[1], {"testament3-sha1": row[2]}))
230
return format("commit", row)
453
231
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
454
232
if row is not None:
455
return ("blob", row)
233
return format("blob", row)
456
234
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
457
235
if row is not None:
458
return ("tree", row)
236
return format("tree", row)
459
237
raise KeyError(sha)
460
238
461
239
def revids(self):
462
240
"""List the revision ids known."""
463
return (row for (row,) in self.db.execute("select revid from commits"))
241
for row in self.db.execute("select revid from commits").fetchall():
242
yield row[0].encode("utf-8")
464
243
465
244
def sha1s(self):
466
245
"""List the SHA1s."""
467
246
for table in ("blobs", "commits", "trees"):
468
for (sha,) in self.db.execute("select sha1 from %s" % table):
469
yield sha
470
471
472
class TdbCacheUpdater(CacheUpdater):
473
"""Cache updater for tdb-based caches."""
474
475
def __init__(self, cache, rev):
476
self.cache = cache
477
self.db = cache.idmap.db
478
self.revid = rev.revision_id
479
self.parent_revids = rev.parent_ids
480
self._commit = None
481
self._entries = []
482
483
def add_object(self, obj, ie, path):
484
sha = obj.sha().digest()
485
if obj.type_name == "commit":
486
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
487
assert type(ie) is dict, "was %r" % ie
488
type_data = (self.revid, obj.tree, ie["testament3-sha1"])
489
self._commit = obj
490
elif obj.type_name == "blob":
491
if ie is None:
492
return
493
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
494
type_data = (ie.file_id, ie.revision)
495
elif obj.type_name == "tree":
496
if ie is None:
497
return
498
type_data = (ie.file_id, self.revid)
499
else:
500
raise AssertionError
501
self.db["git\0" + sha] = "\0".join((obj.type_name, ) + type_data)
502
503
def finish(self):
504
if self._commit is None:
505
raise AssertionError("No commit object added")
506
return self._commit
507
508
509
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
510
511
class TdbGitCacheFormat(BzrGitCacheFormat):
512
"""Cache format for tdb-based caches."""
513
514
def get_format_string(self):
515
return 'bzr-git sha map version 3 using tdb\n'
516
517
def open(self, transport):
518
try:
519
basepath = transport.local_abspath(".")
520
except bzrlib.errors.NotLocalUrl:
521
basepath = get_cache_dir()
522
try:
523
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
524
except ImportError:
525
raise ImportError(
526
"Unable to open existing bzr-git cache because 'tdb' is not "
527
"installed.")
247
for row in self.db.execute("select sha1 from %s" % table).fetchall():
248
yield row[0].encode("utf-8")
249
250
251
TDB_MAP_VERSION = 2
528
252
529
253
530
254
class TdbGitShaMap(GitShaMap):
538
262
"blob fileid revid" -> "<sha1>"
539
263
"""
540
264
541
TDB_MAP_VERSION = 3
542
TDB_HASH_SIZE = 50000
543
544
265
def __init__(self, path=None):
545
266
import tdb
546
267
self.path = path
548
269
self.db = {}
549
270
else:
550
271
if not mapdbs().has_key(path):
551
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
272
mapdbs()[path] = tdb.open(path, 0, tdb.DEFAULT,
552
273
os.O_RDWR|os.O_CREAT)
553
self.db = mapdbs()[path]
554
try:
555
if int(self.db["version"]) not in (2, 3):
274
self.db = mapdbs()[path]
275
if not "version" in self.db:
276
self.db["version"] = str(TDB_MAP_VERSION)
277
else:
278
if int(self.db["version"]) != TDB_MAP_VERSION:
556
279
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
557
self.db["version"], self.TDB_MAP_VERSION)
280
self.db["version"], TDB_MAP_VERSION)
558
281
self.db.clear()
559
except KeyError:
282
self.db["version"] = str(TDB_MAP_VERSION)
283
284
@classmethod
285
def from_repository(cls, repository):
286
try:
287
transport = getattr(repository, "_transport", None)
288
if transport is not None:
289
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
290
except bzrlib.errors.NotLocalUrl:
560
291
pass
561
self.db["version"] = str(self.TDB_MAP_VERSION)
562
563
def start_write_group(self):
564
"""Start writing changes."""
565
self.db.transaction_start()
566
567
def commit_write_group(self):
568
"""Commit any pending changes."""
569
self.db.transaction_commit()
570
571
def abort_write_group(self):
572
"""Abort any pending changes."""
573
self.db.transaction_cancel()
574
575
def __repr__(self):
576
return "%s(%r)" % (self.__class__.__name__, self.path)
292
from bzrlib.config import config_dir
293
return cls(os.path.join(config_dir(), "remote-git.tdb"))
577
294
578
295
def lookup_commit(self, revid):
579
296
return sha_to_hex(self.db["commit\0" + revid][:20])
580
297
581
def lookup_blob_id(self, fileid, revision):
582
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
298
def commit(self):
299
pass
300
301
def add_entry(self, sha, type, type_data):
302
"""Add a new entry to the database.
303
"""
304
self.db["git\0" + hex_to_sha(sha)] = "\0".join((type, type_data[0], type_data[1]))
305
if type == "commit":
306
self.db["commit\0" + type_data[0]] = "\0".join((hex_to_sha(sha), type_data[1]))
307
else:
308
self.db["\0".join((type, type_data[0], type_data[1]))] = hex_to_sha(sha)
309
310
def lookup_tree(self, fileid, revid):
311
return sha_to_hex(self.db["\0".join(("tree", fileid, revid))])
312
313
def lookup_blob(self, fileid, revid):
314
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
583
315
584
316
def lookup_git_sha(self, sha):
585
317
"""Lookup a Git sha in the database.
586
318
587
319
:param sha: Git object sha
588
320
:return: (type, type_data) with type_data:
589
commit: revid, tree sha
590
blob: fileid, revid
591
tree: fileid, revid
321
revision: revid, tree sha
592
322
"""
593
if len(sha) == 40:
594
sha = hex_to_sha(sha)
595
data = self.db["git\0" + sha].split("\0")
596
if data[0] == "commit":
597
if len(data) == 3:
598
return (data[0], (data[1], data[2], {}))
599
else:
600
return (data[0], (data[1], data[2], {"testament3-sha1": data[3]}))
601
else:
602
return (data[0], tuple(data[1:]))
603
604
def missing_revisions(self, revids):
605
ret = set()
606
for revid in revids:
607
if self.db.get("commit\0" + revid) is None:
608
ret.add(revid)
609
return ret
323
data = self.db["git\0" + hex_to_sha(sha)].split("\0")
324
return (data[0], (data[1], data[2]))
610
325
611
326
def revids(self):
612
327
"""List the revision ids known."""
619
334
for key in self.db.iterkeys():
620
335
if key.startswith("git\0"):
621
336
yield sha_to_hex(key[4:])
622
623
624
class VersionedFilesContentCache(ContentCache):
625
626
def __init__(self, vf):
627
self._vf = vf
628
629
def add(self, obj):
630
self._vf.insert_record_stream(
631
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
632
obj.as_legacy_object_chunks())])
633
634
def __getitem__(self, sha):
635
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
636
entry = stream.next()
637
if entry.storage_kind == 'absent':
638
raise KeyError(sha)
639
return ShaFile._parse_legacy_object(entry.get_bytes_as('fulltext'))
640
641
642
class GitObjectStoreContentCache(ContentCache):
643
644
def __init__(self, store):
645
self.store = store
646
647
def add_multi(self, objs):
648
self.store.add_objects(objs)
649
650
def add(self, obj, path):
651
self.store.add_object(obj)
652
653
def __getitem__(self, sha):
654
return self.store[sha]
655
656
657
class IndexCacheUpdater(CacheUpdater):
658
659
def __init__(self, cache, rev):
660
self.cache = cache
661
self.revid = rev.revision_id
662
self.parent_revids = rev.parent_ids
663
self._commit = None
664
self._entries = []
665
self._cache_objs = set()
666
667
def add_object(self, obj, ie, path):
668
if obj.type_name == "commit":
669
self._commit = obj
670
assert type(ie) is dict
671
self.cache.idmap._add_git_sha(obj.id, "commit",
672
(self.revid, obj.tree, ie))
673
self.cache.idmap._add_node(("commit", self.revid, "X"),
674
" ".join((obj.id, obj.tree)))
675
self._cache_objs.add((obj, path))
676
elif obj.type_name == "blob":
677
self.cache.idmap._add_git_sha(obj.id, "blob",
678
(ie.file_id, ie.revision))
679
self.cache.idmap._add_node(("blob", ie.file_id, ie.revision), obj.id)
680
if ie.kind == "symlink":
681
self._cache_objs.add((obj, path))
682
elif obj.type_name == "tree":
683
self.cache.idmap._add_git_sha(obj.id, "tree",
684
(ie.file_id, self.revid))
685
self._cache_objs.add((obj, path))
686
else:
687
raise AssertionError
688
689
def finish(self):
690
self.cache.content_cache.add_multi(self._cache_objs)
691
return self._commit
692
693
694
class IndexBzrGitCache(BzrGitCache):
695
696
def __init__(self, transport=None):
697
mapper = versionedfile.ConstantMapper("trees")
698
shamap = IndexGitShaMap(transport.clone('index'))
699
#trees_store = knit.make_file_factory(True, mapper)(transport)
700
#content_cache = VersionedFilesContentCache(trees_store)
701
from bzrlib.plugins.git.transportgit import TransportObjectStore
702
store = TransportObjectStore(transport.clone('objects'))
703
content_cache = GitObjectStoreContentCache(store)
704
super(IndexBzrGitCache, self).__init__(shamap, content_cache,
705
IndexCacheUpdater)
706
707
708
class IndexGitCacheFormat(BzrGitCacheFormat):
709
710
def get_format_string(self):
711
return 'bzr-git sha map with git object cache version 1\n'
712
713
def initialize(self, transport):
714
super(IndexGitCacheFormat, self).initialize(transport)
715
transport.mkdir('index')
716
transport.mkdir('objects')
717
from bzrlib.plugins.git.transportgit import TransportObjectStore
718
TransportObjectStore.init(transport.clone('objects'))
719
720
def open(self, transport):
721
return IndexBzrGitCache(transport)
722
723
724
class IndexGitShaMap(GitShaMap):
725
"""SHA Map that uses the Bazaar APIs to store a cache.
726
727
BTree Index file with the following contents:
728
729
("git", <sha1>) -> "<type> <type-data1> <type-data2>"
730
("commit", <revid>) -> "<sha1> <tree-id>"
731
("blob", <fileid>, <revid>) -> <sha1>
732
733
"""
734
735
def __init__(self, transport=None):
736
if transport is None:
737
self._transport = None
738
self._index = _mod_index.InMemoryGraphIndex(0, key_elements=3)
739
self._builder = self._index
740
else:
741
self._builder = None
742
self._transport = transport
743
self._index = _mod_index.CombinedGraphIndex([])
744
for name in self._transport.list_dir("."):
745
if not name.endswith(".rix"):
746
continue
747
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
748
self._transport.stat(name).st_size)
749
self._index.insert_index(0, x)
750
751
@classmethod
752
def from_repository(cls, repository):
753
transport = getattr(repository, "_transport", None)
754
if transport is not None:
755
try:
756
transport.mkdir('git')
757
except bzrlib.errors.FileExists:
758
pass
759
return cls(transport.clone('git'))
760
from bzrlib.transport import get_transport
761
return cls(get_transport(get_cache_dir()))
762
763
def __repr__(self):
764
if self._transport is not None:
765
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
766
else:
767
return "%s()" % (self.__class__.__name__)
768
769
def repack(self):
770
assert self._builder is None
771
self.start_write_group()
772
for _, key, value in self._index.iter_all_entries():
773
self._builder.add_node(key, value)
774
to_remove = []
775
for name in self._transport.list_dir('.'):
776
if name.endswith('.rix'):
777
to_remove.append(name)
778
self.commit_write_group()
779
del self._index.indices[1:]
780
for name in to_remove:
781
self._transport.rename(name, name + '.old')
782
783
def start_write_group(self):
784
assert self._builder is None
785
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
786
self._name = osutils.sha()
787
788
def commit_write_group(self):
789
assert self._builder is not None
790
stream = self._builder.finish()
791
name = self._name.hexdigest() + ".rix"
792
size = self._transport.put_file(name, stream)
793
index = _mod_btree_index.BTreeGraphIndex(self._transport, name, size)
794
self._index.insert_index(0, index)
795
self._builder = None
796
self._name = None
797
798
def abort_write_group(self):
799
assert self._builder is not None
800
self._builder = None
801
self._name = None
802
803
def _add_node(self, key, value):
804
try:
805
self._builder.add_node(key, value)
806
except bzrlib.errors.BadIndexDuplicateKey:
807
# Multiple bzr objects can have the same contents
808
return True
809
else:
810
return False
811
812
def _get_entry(self, key):
813
entries = self._index.iter_entries([key])
814
try:
815
return entries.next()[2]
816
except StopIteration:
817
if self._builder is None:
818
raise KeyError
819
entries = self._builder.iter_entries([key])
820
try:
821
return entries.next()[2]
822
except StopIteration:
823
raise KeyError
824
825
def _iter_keys_prefix(self, prefix):
826
for entry in self._index.iter_entries_prefix([prefix]):
827
yield entry[1]
828
if self._builder is not None:
829
for entry in self._builder.iter_entries_prefix([prefix]):
830
yield entry[1]
831
832
def lookup_commit(self, revid):
833
return self._get_entry(("commit", revid, "X"))[:40]
834
835
def _add_git_sha(self, hexsha, type, type_data):
836
if hexsha is not None:
837
self._name.update(hexsha)
838
if type == "commit":
839
td = (type_data[0], type_data[1], type_data[2]["testament3-sha1"])
840
else:
841
td = type_data
842
self._add_node(("git", hexsha, "X"), " ".join((type,) + td))
843
else:
844
# This object is not represented in Git - perhaps an empty
845
# directory?
846
self._name.update(type + " ".join(type_data))
847
848
def lookup_blob_id(self, fileid, revision):
849
return self._get_entry(("blob", fileid, revision))
850
851
def lookup_git_sha(self, sha):
852
if len(sha) == 20:
853
sha = sha_to_hex(sha)
854
data = self._get_entry(("git", sha, "X")).split(" ", 3)
855
if data[0] == "commit":
856
return ("commit", (data[1], data[2], {"testament3-sha1": data[3]}))
857
else:
858
return (data[0], tuple(data[1:]))
859
860
def revids(self):
861
"""List the revision ids known."""
862
for key in self._iter_keys_prefix(("commit", None, None)):
863
yield key[1]
864
865
def missing_revisions(self, revids):
866
"""Return set of all the revisions that are not present."""
867
missing_revids = set(revids)
868
for _, key, value in self._index.iter_entries((
869
("commit", revid, "X") for revid in revids)):
870
missing_revids.remove(key[1])
871
return missing_revids
872
873
def sha1s(self):
874
"""List the SHA1s."""
875
for key in self._iter_keys_prefix(("git", None, None)):
876
yield key[1]
877
878
879
formats = registry.Registry()
880
formats.register(TdbGitCacheFormat().get_format_string(),
881
TdbGitCacheFormat())
882
formats.register(SqliteGitCacheFormat().get_format_string(),
883
SqliteGitCacheFormat())
884
formats.register(IndexGitCacheFormat().get_format_string(),
885
IndexGitCacheFormat())
886
# In the future, this will become the default:
887
# formats.register('default', IndexGitCacheFormat())
888
try:
889
import tdb
890
except ImportError:
891
formats.register('default', SqliteGitCacheFormat())
892
else:
893
formats.register('default', TdbGitCacheFormat())
894
895
896
897
def migrate_ancient_formats(repo_transport):
898
# Prefer migrating git.db over git.tdb, since the latter may not
899
# be openable on some platforms.
900
if repo_transport.has("git.db"):
901
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
902
repo_transport.rename("git.db", "git/idmap.db")
903
elif repo_transport.has("git.tdb"):
904
TdbGitCacheFormat().initialize(repo_transport.clone("git"))
905
repo_transport.rename("git.tdb", "git/idmap.tdb")
906
907
908
def remove_readonly_transport_decorator(transport):
909
if transport.is_readonly():
910
return transport._decorated
911
return transport
912
913
914
def from_repository(repository):
915
"""Open a cache file for a repository.
916
917
If the repository is remote and there is no transport available from it
918
this will use a local file in the users cache directory
919
(typically ~/.cache/bazaar/git/)
920
921
:param repository: A repository object
922
"""
923
repo_transport = getattr(repository, "_transport", None)
924
if repo_transport is not None:
925
# Migrate older cache formats
926
repo_transport = remove_readonly_transport_decorator(repo_transport)
927
try:
928
repo_transport.mkdir("git")
929
except bzrlib.errors.FileExists:
930
pass
931
else:
932
migrate_ancient_formats(repo_transport)
933
return BzrGitCacheFormat.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0