To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to cache.py
- browse files at revision 0.200.990
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.990
- Committer: Jelmer Vernooij
- Date: 2010-08-03 19:47:30 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100803194730-2llh64hneegnseqp
Support 'add' in empty directory.
- files added:
- hg.py
- files renamed:
- shamap.py => cache.py
- tests/test_shamap.py => tests/test_cache.py
- files modified:
- HACKING
added
removed
cache.py
23
23
import os
24
24
import threading
25
25
26
from dulwich.objects import (
27
ShaFile,
28
)
29
26
30
import bzrlib
27
31
from bzrlib import (
32
btree_index as _mod_btree_index,
33
index as _mod_index,
34
osutils,
35
registry,
28
36
trace,
37
versionedfile,
29
38
)
30
from bzrlib.errors import (
31
NoSuchRevision,
39
from bzrlib.transport import (
40
get_transport,
32
41
)
33
42
34
43
45
54
return ret
46
55
47
56
57
def get_remote_cache_transport():
58
return get_transport(get_cache_dir())
59
60
48
61
def check_pysqlite_version(sqlite3):
49
62
"""Check that sqlite library is compatible.
50
63
51
64
"""
52
if (sqlite3.sqlite_version_info[0] < 3 or
53
(sqlite3.sqlite_version_info[0] == 3 and
65
if (sqlite3.sqlite_version_info[0] < 3 or
66
(sqlite3.sqlite_version_info[0] == 3 and
54
67
sqlite3.sqlite_version_info[1] < 3)):
55
warning('Needs at least sqlite 3.3.x')
68
trace.warning('Needs at least sqlite 3.3.x')
56
69
raise bzrlib.errors.BzrError("incompatible sqlite library")
57
70
58
71
try:
59
72
try:
60
73
import sqlite3
61
74
check_pysqlite_version(sqlite3)
62
except (ImportError, bzrlib.errors.BzrError), e:
75
except (ImportError, bzrlib.errors.BzrError), e:
63
76
from pysqlite2 import dbapi2 as sqlite3
64
77
check_pysqlite_version(sqlite3)
65
78
except:
66
warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
79
trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
67
80
'module')
68
81
raise bzrlib.errors.BzrError("missing sqlite library")
69
82
81
94
class GitShaMap(object):
82
95
"""Git<->Bzr revision id mapping database."""
83
96
84
def add_entry(self, sha, type, type_data):
85
"""Add a new entry to the database.
86
"""
87
raise NotImplementedError(self.add_entry)
88
89
def add_entries(self, entries):
90
"""Add multiple new entries to the database.
91
"""
92
for e in entries:
93
self.add_entry(*e)
94
95
def lookup_tree(self, fileid, revid):
96
"""Lookup the SHA of a git tree."""
97
raise NotImplementedError(self.lookup_tree)
98
99
def lookup_blob(self, fileid, revid):
100
"""Lookup a blob by the fileid it has in a bzr revision."""
101
raise NotImplementedError(self.lookup_blob)
102
103
97
def lookup_git_sha(self, sha):
104
98
"""Lookup a Git sha in the database.
105
106
99
:param sha: Git object sha
107
100
:return: (type, type_data) with type_data:
108
101
revision: revid, tree sha
109
102
"""
110
103
raise NotImplementedError(self.lookup_git_sha)
111
104
105
def lookup_blob_id(self, file_id, revision):
106
"""Retrieve a Git blob SHA by file id.
107
108
:param file_id: File id of the file/symlink
109
:param revision: revision in which the file was last changed.
110
"""
111
raise NotImplementedError(self.lookup_blob_id)
112
113
def lookup_tree_id(self, file_id, revision):
114
"""Retrieve a Git tree SHA by file id.
115
"""
116
raise NotImplementedError(self.lookup_tree_id)
117
112
118
def revids(self):
113
119
"""List the revision ids known."""
114
120
raise NotImplementedError(self.revids)
115
121
116
def sha1s(Self):
122
def missing_revisions(self, revids):
123
"""Return set of all the revisions that are not present."""
124
present_revids = set(self.revids())
125
if not isinstance(revids, set):
126
revids = set(revids)
127
return revids - present_revids
128
129
def sha1s(self):
117
130
"""List the SHA1s."""
118
131
raise NotImplementedError(self.sha1s)
119
132
120
def commit(self):
133
def start_write_group(self):
134
"""Start writing changes."""
135
136
def commit_write_group(self):
121
137
"""Commit any pending changes."""
122
138
139
def abort_write_group(self):
140
"""Abort any pending changes."""
141
142
143
class ContentCache(object):
144
"""Object that can cache Git objects."""
145
146
def add(self, object):
147
"""Add an object."""
148
raise NotImplementedError(self.add)
149
150
def add_multi(self, objects):
151
"""Add multiple objects."""
152
for obj in objects:
153
self.add(obj)
154
155
def __getitem__(self, sha):
156
"""Retrieve an item, by SHA."""
157
raise NotImplementedError(self.__getitem__)
158
159
160
class BzrGitCacheFormat(object):
161
"""Bazaar-Git Cache Format."""
162
163
def get_format_string(self):
164
"""Return a single-line unique format string for this cache format."""
165
raise NotImplementedError(self.get_format_string)
166
167
def open(self, transport):
168
"""Open this format on a transport."""
169
raise NotImplementedError(self.open)
170
171
def initialize(self, transport):
172
"""Create a new instance of this cache format at transport."""
173
transport.put_bytes('format', self.get_format_string())
174
175
@classmethod
176
def from_transport(self, transport):
177
"""Open a cache file present on a transport, or initialize one.
178
179
:param transport: Transport to use
180
:return: A BzrGitCache instance
181
"""
182
try:
183
format_name = transport.get_bytes('format')
184
format = formats.get(format_name)
185
except bzrlib.errors.NoSuchFile:
186
format = formats.get('default')
187
format.initialize(transport)
188
return format.open(transport)
189
190
@classmethod
191
def from_repository(cls, repository):
192
"""Open a cache file for a repository.
193
194
This will use the repository's transport to store the cache file, or
195
use the users global cache directory if the repository has no
196
transport associated with it.
197
198
:param repository: Repository to open the cache for
199
:return: A `BzrGitCache`
200
"""
201
repo_transport = getattr(repository, "_transport", None)
202
if repo_transport is not None:
203
# Even if we don't write to this repo, we should be able
204
# to update its cache.
205
repo_transport = remove_readonly_transport_decorator(repo_transport)
206
try:
207
repo_transport.mkdir('git')
208
except bzrlib.errors.FileExists:
209
pass
210
transport = repo_transport.clone('git')
211
else:
212
transport = get_remote_cache_transport()
213
return cls.from_transport(transport)
214
215
216
class CacheUpdater(object):
217
"""Base class for objects that can update a bzr-git cache."""
218
219
def add_object(self, obj, ie, path):
220
raise NotImplementedError(self.add_object)
221
222
def finish(self):
223
raise NotImplementedError(self.finish)
224
225
226
class BzrGitCache(object):
227
"""Caching backend."""
228
229
def __init__(self, idmap, content_cache, cache_updater_klass):
230
self.idmap = idmap
231
self.content_cache = content_cache
232
self._cache_updater_klass = cache_updater_klass
233
234
def get_updater(self, rev):
235
"""Update an object that implements the CacheUpdater interface for
236
updating this cache.
237
"""
238
return self._cache_updater_klass(self, rev)
239
240
241
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), None, DictCacheUpdater)
242
243
244
class DictCacheUpdater(CacheUpdater):
245
"""Cache updater for dict-based caches."""
246
247
def __init__(self, cache, rev):
248
self.cache = cache
249
self.revid = rev.revision_id
250
self.parent_revids = rev.parent_ids
251
self._commit = None
252
self._entries = []
253
254
def add_object(self, obj, ie, path):
255
if obj.type_name == "commit":
256
self._commit = obj
257
assert ie is None
258
type_data = (self.revid, self._commit.tree)
259
self.cache.idmap._by_revid[self.revid] = obj.id
260
elif obj.type_name in ("blob", "tree"):
261
if ie is not None:
262
if obj.type_name == "blob":
263
revision = ie.revision
264
else:
265
revision = self.revid
266
type_data = (ie.file_id, revision)
267
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] =\
268
obj.id
269
else:
270
raise AssertionError
271
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
272
273
def finish(self):
274
if self._commit is None:
275
raise AssertionError("No commit object added")
276
return self._commit
277
123
278
124
279
class DictGitShaMap(GitShaMap):
280
"""Git SHA map that uses a dictionary."""
125
281
126
282
def __init__(self):
127
self.dict = {}
283
self._by_sha = {}
284
self._by_fileid = {}
285
self._by_revid = {}
128
286
129
def add_entry(self, sha, type, type_data):
130
self.dict[sha] = (type, type_data)
287
def lookup_blob_id(self, fileid, revision):
288
return self._by_fileid[revision][fileid]
131
289
132
290
def lookup_git_sha(self, sha):
133
return self.dict[sha]
134
135
def lookup_tree(self, fileid, revid):
136
for k, v in self.dict.iteritems():
137
if v == ("tree", (fileid, revid)):
138
return k
139
raise KeyError((fileid, revid))
140
141
def lookup_blob(self, fileid, revid):
142
for k, v in self.dict.iteritems():
143
if v == ("blob", (fileid, revid)):
144
return k
145
raise KeyError((fileid, revid))
291
return self._by_sha[sha]
292
293
def lookup_tree_id(self, fileid, revision):
294
return self._by_fileid[revision][fileid]
295
296
def lookup_commit(self, revid):
297
return self._by_revid[revid]
146
298
147
299
def revids(self):
148
for key, (type, type_data) in self.dict.iteritems():
300
for key, (type, type_data) in self._by_sha.iteritems():
149
301
if type == "commit":
150
302
yield type_data[0]
151
303
152
304
def sha1s(self):
153
return self.dict.iterkeys()
305
return self._by_sha.iterkeys()
306
307
308
class SqliteCacheUpdater(CacheUpdater):
309
310
def __init__(self, cache, rev):
311
self.cache = cache
312
self.db = self.cache.idmap.db
313
self.revid = rev.revision_id
314
self._commit = None
315
self._trees = []
316
self._blobs = []
317
318
def add_object(self, obj, ie, path):
319
if obj.type_name == "commit":
320
self._commit = obj
321
assert ie is None
322
elif obj.type_name == "tree":
323
if ie is not None:
324
self._trees.append((obj.id, ie.file_id, self.revid))
325
elif obj.type_name == "blob":
326
if ie is not None:
327
self._blobs.append((obj.id, ie.file_id, ie.revision))
328
else:
329
raise AssertionError
330
331
def finish(self):
332
if self._commit is None:
333
raise AssertionError("No commit object added")
334
self.db.executemany(
335
"replace into trees (sha1, fileid, revid) values (?, ?, ?)",
336
self._trees)
337
self.db.executemany(
338
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
339
self._blobs)
340
self.db.execute(
341
"replace into commits (sha1, revid, tree_sha) values (?, ?, ?)",
342
(self._commit.id, self.revid, self._commit.tree))
343
return self._commit
344
345
346
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), None, SqliteCacheUpdater)
347
348
349
class SqliteGitCacheFormat(BzrGitCacheFormat):
350
351
def get_format_string(self):
352
return 'bzr-git sha map version 1 using sqlite\n'
353
354
def open(self, transport):
355
try:
356
basepath = transport.local_abspath(".")
357
except bzrlib.errors.NotLocalUrl:
358
basepath = get_cache_dir()
359
return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))
154
360
155
361
156
362
class SqliteGitShaMap(GitShaMap):
363
"""Bazaar GIT Sha map that uses a sqlite database for storage."""
157
364
158
365
def __init__(self, path=None):
159
366
self.path = path
162
369
else:
163
370
if not mapdbs().has_key(path):
164
371
mapdbs()[path] = sqlite3.connect(path)
165
self.db = mapdbs()[path]
372
self.db = mapdbs()[path]
373
self.db.text_factory = str
166
374
self.db.executescript("""
167
create table if not exists commits(sha1 text, revid text, tree_sha text);
375
create table if not exists commits(
376
sha1 text not null check(length(sha1) == 40),
377
revid text not null,
378
tree_sha text not null check(length(tree_sha) == 40)
379
);
168
380
create index if not exists commit_sha1 on commits(sha1);
169
381
create unique index if not exists commit_revid on commits(revid);
170
create table if not exists blobs(sha1 text, fileid text, revid text);
382
create table if not exists blobs(
383
sha1 text not null check(length(sha1) == 40),
384
fileid text not null,
385
revid text not null
386
);
171
387
create index if not exists blobs_sha1 on blobs(sha1);
172
388
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
173
create table if not exists trees(sha1 text, fileid text, revid text);
174
create index if not exists trees_sha1 on trees(sha1);
389
create table if not exists trees(
390
sha1 text unique not null check(length(sha1) == 40),
391
fileid text not null,
392
revid text not null
393
);
394
create unique index if not exists trees_sha1 on trees(sha1);
175
395
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
176
396
""")
177
397
178
@classmethod
179
def from_repository(cls, repository):
180
try:
181
transport = getattr(repository, "_transport", None)
182
if transport is not None:
183
return cls(os.path.join(transport.local_abspath("."), "git.db"))
184
except bzrlib.errors.NotLocalUrl:
185
pass
186
return cls(os.path.join(get_cache_dir(), "remote.db"))
187
398
def __repr__(self):
399
return "%s(%r)" % (self.__class__.__name__, self.path)
400
188
401
def lookup_commit(self, revid):
189
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
402
cursor = self.db.execute("select sha1 from commits where revid = ?",
403
(revid,))
404
row = cursor.fetchone()
190
405
if row is not None:
191
return row[0].encode("utf-8")
406
return row[0]
192
407
raise KeyError
193
408
194
def commit(self):
409
def commit_write_group(self):
195
410
self.db.commit()
196
411
197
def add_entries(self, entries):
198
trees = []
199
blobs = []
200
for sha, type, type_data in entries:
201
assert isinstance(type_data[0], str)
202
assert isinstance(type_data[1], str)
203
entry = (sha.decode("utf-8"), type_data[0].decode("utf-8"),
204
type_data[1].decode("utf-8"))
205
if type == "tree":
206
trees.append(entry)
207
elif type == "blob":
208
blobs.append(entry)
209
else:
210
raise AssertionError
211
if trees:
212
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
213
if blobs:
214
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
215
216
217
def add_entry(self, sha, type, type_data):
218
"""Add a new entry to the database.
219
"""
220
assert isinstance(type_data, tuple)
221
assert isinstance(sha, str), "type was %r" % sha
222
if type == "commit":
223
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
224
elif type in ("blob", "tree"):
225
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
226
else:
227
raise AssertionError("Unknown type %s" % type)
228
229
def lookup_tree(self, fileid, revid):
230
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid,revid)).fetchone()
231
if row is None:
232
raise KeyError((fileid, revid))
233
return row[0].encode("utf-8")
234
235
def lookup_blob(self, fileid, revid):
236
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
237
if row is None:
238
raise KeyError((fileid, revid))
239
return row[0].encode("utf-8")
412
def lookup_blob_id(self, fileid, revision):
413
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
414
if row is not None:
415
return row[0]
416
raise KeyError(fileid)
417
418
def lookup_tree_id(self, fileid, revision):
419
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
420
if row is not None:
421
return row[0]
422
raise KeyError(fileid)
240
423
241
424
def lookup_git_sha(self, sha):
242
425
"""Lookup a Git sha in the database.
245
428
:return: (type, type_data) with type_data:
246
429
revision: revid, tree sha
247
430
"""
248
def format(type, row):
249
return (type, (row[0].encode("utf-8"), row[1].encode("utf-8")))
250
431
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
251
432
if row is not None:
252
return format("commit", row)
433
return ("commit", row)
253
434
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
254
435
if row is not None:
255
return format("blob", row)
436
return ("blob", row)
256
437
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
257
438
if row is not None:
258
return format("tree", row)
439
return ("tree", row)
259
440
raise KeyError(sha)
260
441
261
442
def revids(self):
262
443
"""List the revision ids known."""
263
for row in self.db.execute("select revid from commits").fetchall():
264
yield row[0].encode("utf-8")
444
return (row for (row,) in self.db.execute("select revid from commits"))
265
445
266
446
def sha1s(self):
267
447
"""List the SHA1s."""
268
448
for table in ("blobs", "commits", "trees"):
269
for row in self.db.execute("select sha1 from %s" % table).fetchall():
270
yield row[0].encode("utf-8")
271
272
273
TDB_MAP_VERSION = 2
274
TDB_HASH_SIZE = 10000
449
for (sha,) in self.db.execute("select sha1 from %s" % table):
450
yield sha
451
452
453
class TdbCacheUpdater(CacheUpdater):
454
"""Cache updater for tdb-based caches."""
455
456
def __init__(self, cache, rev):
457
self.cache = cache
458
self.db = cache.idmap.db
459
self.revid = rev.revision_id
460
self.parent_revids = rev.parent_ids
461
self._commit = None
462
self._entries = []
463
464
def add_object(self, obj, ie, path):
465
sha = obj.sha().digest()
466
if obj.type_name == "commit":
467
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
468
type_data = (self.revid, obj.tree)
469
self._commit = obj
470
assert ie is None
471
elif obj.type_name == "blob":
472
if ie is None:
473
return
474
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
475
type_data = (ie.file_id, ie.revision)
476
elif obj.type_name == "tree":
477
if ie is None:
478
return
479
type_data = (ie.file_id, self.revid)
480
else:
481
raise AssertionError
482
self.db["git\0" + sha] = "\0".join((obj.type_name, ) + type_data)
483
484
def finish(self):
485
if self._commit is None:
486
raise AssertionError("No commit object added")
487
return self._commit
488
489
490
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
491
492
class TdbGitCacheFormat(BzrGitCacheFormat):
493
"""Cache format for tdb-based caches."""
494
495
def get_format_string(self):
496
return 'bzr-git sha map version 3 using tdb\n'
497
498
def open(self, transport):
499
try:
500
basepath = transport.local_abspath(".")
501
except bzrlib.errors.NotLocalUrl:
502
basepath = get_cache_dir()
503
try:
504
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
505
except ImportError:
506
raise ImportError(
507
"Unable to open existing bzr-git cache because 'tdb' is not "
508
"installed.")
275
509
276
510
277
511
class TdbGitShaMap(GitShaMap):
285
519
"blob fileid revid" -> "<sha1>"
286
520
"""
287
521
522
TDB_MAP_VERSION = 3
523
TDB_HASH_SIZE = 50000
524
288
525
def __init__(self, path=None):
289
526
import tdb
290
527
self.path = path
292
529
self.db = {}
293
530
else:
294
531
if not mapdbs().has_key(path):
295
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
532
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
296
533
os.O_RDWR|os.O_CREAT)
297
self.db = mapdbs()[path]
298
if not "version" in self.db:
299
self.db["version"] = str(TDB_MAP_VERSION)
300
else:
301
if int(self.db["version"]) != TDB_MAP_VERSION:
534
self.db = mapdbs()[path]
535
try:
536
if int(self.db["version"]) not in (2, 3):
302
537
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
303
self.db["version"], TDB_MAP_VERSION)
538
self.db["version"], self.TDB_MAP_VERSION)
304
539
self.db.clear()
305
self.db["version"] = str(TDB_MAP_VERSION)
306
307
@classmethod
308
def from_repository(cls, repository):
309
try:
310
transport = getattr(repository, "_transport", None)
311
if transport is not None:
312
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
313
except bzrlib.errors.NotLocalUrl:
540
except KeyError:
314
541
pass
315
return cls(os.path.join(get_cache_dir(), "remote.tdb"))
542
self.db["version"] = str(self.TDB_MAP_VERSION)
543
544
def start_write_group(self):
545
"""Start writing changes."""
546
self.db.transaction_start()
547
548
def commit_write_group(self):
549
"""Commit any pending changes."""
550
self.db.transaction_commit()
551
552
def abort_write_group(self):
553
"""Abort any pending changes."""
554
self.db.transaction_cancel()
555
556
def __repr__(self):
557
return "%s(%r)" % (self.__class__.__name__, self.path)
316
558
317
559
def lookup_commit(self, revid):
318
560
return sha_to_hex(self.db["commit\0" + revid][:20])
319
561
320
def commit(self):
321
pass
322
323
def add_entry(self, sha, type, type_data):
324
"""Add a new entry to the database.
325
"""
326
self.db["git\0" + hex_to_sha(sha)] = "\0".join((type, type_data[0], type_data[1]))
327
if type == "commit":
328
self.db["commit\0" + type_data[0]] = "\0".join((hex_to_sha(sha), type_data[1]))
329
else:
330
self.db["\0".join((type, type_data[0], type_data[1]))] = hex_to_sha(sha)
331
332
def lookup_tree(self, fileid, revid):
333
return sha_to_hex(self.db["\0".join(("tree", fileid, revid))])
334
335
def lookup_blob(self, fileid, revid):
336
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
337
562
def lookup_blob_id(self, fileid, revision):
563
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
564
338
565
def lookup_git_sha(self, sha):
339
566
"""Lookup a Git sha in the database.
340
567
342
569
:return: (type, type_data) with type_data:
343
570
revision: revid, tree sha
344
571
"""
345
data = self.db["git\0" + hex_to_sha(sha)].split("\0")
572
if len(sha) == 40:
573
sha = hex_to_sha(sha)
574
data = self.db["git\0" + sha].split("\0")
346
575
return (data[0], (data[1], data[2]))
347
576
577
def missing_revisions(self, revids):
578
ret = set()
579
for revid in revids:
580
if self.db.get("commit\0" + revid) is None:
581
ret.add(revid)
582
return ret
583
348
584
def revids(self):
349
585
"""List the revision ids known."""
350
586
for key in self.db.iterkeys():
356
592
for key in self.db.iterkeys():
357
593
if key.startswith("git\0"):
358
594
yield sha_to_hex(key[4:])
595
596
597
class VersionedFilesContentCache(ContentCache):
598
599
def __init__(self, vf):
600
self._vf = vf
601
602
def add(self, obj):
603
self._vf.insert_record_stream(
604
[versionedfile.ChunkedContentFactory((obj.id,), [], None,
605
obj.as_legacy_object_chunks())])
606
607
def __getitem__(self, sha):
608
stream = self._vf.get_record_stream([(sha,)], 'unordered', True)
609
entry = stream.next()
610
if entry.storage_kind == 'absent':
611
raise KeyError(sha)
612
return ShaFile._parse_legacy_object(entry.get_bytes_as('fulltext'))
613
614
615
class GitObjectStoreContentCache(ContentCache):
616
617
def __init__(self, store):
618
self.store = store
619
620
def add_multi(self, objs):
621
self.store.add_objects(objs)
622
623
def add(self, obj, path):
624
self.store.add_object(obj)
625
626
def __getitem__(self, sha):
627
return self.store[sha]
628
629
630
class IndexCacheUpdater(CacheUpdater):
631
632
def __init__(self, cache, rev):
633
self.cache = cache
634
self.revid = rev.revision_id
635
self.parent_revids = rev.parent_ids
636
self._commit = None
637
self._entries = []
638
self._cache_objs = set()
639
640
def add_object(self, obj, ie, path):
641
if obj.type_name == "commit":
642
self._commit = obj
643
assert ie is None
644
self.cache.idmap._add_git_sha(obj.id, "commit",
645
(self.revid, obj.tree))
646
self.cache.idmap._add_node(("commit", self.revid, "X"),
647
" ".join((obj.id, obj.tree)))
648
self._cache_objs.add((obj, path))
649
elif obj.type_name == "blob":
650
self.cache.idmap._add_git_sha(obj.id, "blob",
651
(ie.file_id, ie.revision))
652
self.cache.idmap._add_node(("blob", ie.file_id, ie.revision), obj.id)
653
if ie.kind == "symlink":
654
self._cache_objs.add((obj, path))
655
elif obj.type_name == "tree":
656
self.cache.idmap._add_git_sha(obj.id, "tree",
657
(ie.file_id, self.revid))
658
self._cache_objs.add((obj, path))
659
else:
660
raise AssertionError
661
662
def finish(self):
663
self.cache.content_cache.add_multi(self._cache_objs)
664
return self._commit
665
666
667
class IndexBzrGitCache(BzrGitCache):
668
669
def __init__(self, transport=None):
670
mapper = versionedfile.ConstantMapper("trees")
671
shamap = IndexGitShaMap(transport.clone('index'))
672
#trees_store = knit.make_file_factory(True, mapper)(transport)
673
#content_cache = VersionedFilesContentCache(trees_store)
674
from bzrlib.plugins.git.transportgit import TransportObjectStore
675
store = TransportObjectStore(transport.clone('objects'))
676
content_cache = GitObjectStoreContentCache(store)
677
super(IndexBzrGitCache, self).__init__(shamap, content_cache,
678
IndexCacheUpdater)
679
680
681
class IndexGitCacheFormat(BzrGitCacheFormat):
682
683
def get_format_string(self):
684
return 'bzr-git sha map with git object cache version 1\n'
685
686
def initialize(self, transport):
687
super(IndexGitCacheFormat, self).initialize(transport)
688
transport.mkdir('index')
689
transport.mkdir('objects')
690
from bzrlib.plugins.git.transportgit import TransportObjectStore
691
TransportObjectStore.init(transport.clone('objects'))
692
693
def open(self, transport):
694
return IndexBzrGitCache(transport)
695
696
697
class IndexGitShaMap(GitShaMap):
698
"""SHA Map that uses the Bazaar APIs to store a cache.
699
700
BTree Index file with the following contents:
701
702
("git", <sha1>) -> "<type> <type-data1> <type-data2>"
703
("commit", <revid>) -> "<sha1> <tree-id>"
704
("blob", <fileid>, <revid>) -> <sha1>
705
706
"""
707
708
def __init__(self, transport=None):
709
if transport is None:
710
self._transport = None
711
self._index = _mod_index.InMemoryGraphIndex(0, key_elements=3)
712
self._builder = self._index
713
else:
714
self._builder = None
715
self._transport = transport
716
self._index = _mod_index.CombinedGraphIndex([])
717
for name in self._transport.list_dir("."):
718
if not name.endswith(".rix"):
719
continue
720
x = _mod_btree_index.BTreeGraphIndex(self._transport, name,
721
self._transport.stat(name).st_size)
722
self._index.insert_index(0, x)
723
724
@classmethod
725
def from_repository(cls, repository):
726
transport = getattr(repository, "_transport", None)
727
if transport is not None:
728
try:
729
transport.mkdir('git')
730
except bzrlib.errors.FileExists:
731
pass
732
return cls(transport.clone('git'))
733
from bzrlib.transport import get_transport
734
return cls(get_transport(get_cache_dir()))
735
736
def __repr__(self):
737
if self._transport is not None:
738
return "%s(%r)" % (self.__class__.__name__, self._transport.base)
739
else:
740
return "%s()" % (self.__class__.__name__)
741
742
def repack(self):
743
assert self._builder is None
744
self.start_write_group()
745
for _, key, value in self._index.iter_all_entries():
746
self._builder.add_node(key, value)
747
to_remove = []
748
for name in self._transport.list_dir('.'):
749
if name.endswith('.rix'):
750
to_remove.append(name)
751
self.commit_write_group()
752
del self._index.indices[1:]
753
for name in to_remove:
754
self._transport.rename(name, name + '.old')
755
756
def start_write_group(self):
757
assert self._builder is None
758
self._builder = _mod_btree_index.BTreeBuilder(0, key_elements=3)
759
self._name = osutils.sha()
760
761
def commit_write_group(self):
762
assert self._builder is not None
763
stream = self._builder.finish()
764
name = self._name.hexdigest() + ".rix"
765
size = self._transport.put_file(name, stream)
766
index = _mod_btree_index.BTreeGraphIndex(self._transport, name, size)
767
self._index.insert_index(0, index)
768
self._builder = None
769
self._name = None
770
771
def abort_write_group(self):
772
assert self._builder is not None
773
self._builder = None
774
self._name = None
775
776
def _add_node(self, key, value):
777
try:
778
self._builder.add_node(key, value)
779
except bzrlib.errors.BadIndexDuplicateKey:
780
# Multiple bzr objects can have the same contents
781
return True
782
else:
783
return False
784
785
def _get_entry(self, key):
786
entries = self._index.iter_entries([key])
787
try:
788
return entries.next()[2]
789
except StopIteration:
790
if self._builder is None:
791
raise KeyError
792
entries = self._builder.iter_entries([key])
793
try:
794
return entries.next()[2]
795
except StopIteration:
796
raise KeyError
797
798
def _iter_keys_prefix(self, prefix):
799
for entry in self._index.iter_entries_prefix([prefix]):
800
yield entry[1]
801
if self._builder is not None:
802
for entry in self._builder.iter_entries_prefix([prefix]):
803
yield entry[1]
804
805
def lookup_commit(self, revid):
806
return self._get_entry(("commit", revid, "X"))[:40]
807
808
def _add_git_sha(self, hexsha, type, type_data):
809
if hexsha is not None:
810
self._name.update(hexsha)
811
self._add_node(("git", hexsha, "X"),
812
" ".join((type, type_data[0], type_data[1])))
813
else:
814
# This object is not represented in Git - perhaps an empty
815
# directory?
816
self._name.update(type + " ".join(type_data))
817
818
def lookup_blob_id(self, fileid, revision):
819
return self._get_entry(("blob", fileid, revision))
820
821
def lookup_git_sha(self, sha):
822
if len(sha) == 20:
823
sha = sha_to_hex(sha)
824
data = self._get_entry(("git", sha, "X")).split(" ", 2)
825
return (data[0], (data[1], data[2]))
826
827
def revids(self):
828
"""List the revision ids known."""
829
for key in self._iter_keys_prefix(("commit", None, None)):
830
yield key[1]
831
832
def missing_revisions(self, revids):
833
"""Return set of all the revisions that are not present."""
834
missing_revids = set(revids)
835
for _, key, value in self._index.iter_entries((
836
("commit", revid, "X") for revid in revids)):
837
missing_revids.remove(key[1])
838
return missing_revids
839
840
def sha1s(self):
841
"""List the SHA1s."""
842
for key in self._iter_keys_prefix(("git", None, None)):
843
yield key[1]
844
845
846
formats = registry.Registry()
847
formats.register(TdbGitCacheFormat().get_format_string(),
848
TdbGitCacheFormat())
849
formats.register(SqliteGitCacheFormat().get_format_string(),
850
SqliteGitCacheFormat())
851
formats.register(IndexGitCacheFormat().get_format_string(),
852
IndexGitCacheFormat())
853
# In the future, this will become the default:
854
# formats.register('default', IndexGitCacheFormat())
855
try:
856
import tdb
857
except ImportError:
858
formats.register('default', SqliteGitCacheFormat())
859
else:
860
formats.register('default', TdbGitCacheFormat())
861
862
863
864
def migrate_ancient_formats(repo_transport):
865
# Prefer migrating git.db over git.tdb, since the latter may not
866
# be openable on some platforms.
867
if repo_transport.has("git.db"):
868
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
869
repo_transport.rename("git.db", "git/idmap.db")
870
elif repo_transport.has("git.tdb"):
871
TdbGitCacheFormat().initialize(repo_transport.clone("git"))
872
repo_transport.rename("git.tdb", "git/idmap.tdb")
873
874
875
def remove_readonly_transport_decorator(transport):
876
if transport.is_readonly():
877
return transport._decorated
878
return transport
879
880
881
def from_repository(repository):
882
"""Open a cache file for a repository.
883
884
If the repository is remote and there is no transport available from it
885
this will use a local file in the users cache directory
886
(typically ~/.cache/bazaar/git/)
887
888
:param repository: A repository object
889
"""
890
repo_transport = getattr(repository, "_transport", None)
891
if repo_transport is not None:
892
# Migrate older cache formats
893
repo_transport = remove_readonly_transport_decorator(repo_transport)
894
try:
895
repo_transport.mkdir("git")
896
except bzrlib.errors.FileExists:
897
pass
898
else:
899
migrate_ancient_formats(repo_transport)
900
return BzrGitCacheFormat.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0