To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to shamap.py
- browse files at revision 0.200.730
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.730
- Committer: Jelmer Vernooij
- Date: 2010-03-03 16:40:57 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100303164057-1i8b44wv0y38vn0q
Don't claim control directories can be accessed directly, always open the
branch itself.
branch itself.
- files removed:
- notes/git-serve.txt
- files modified:
- NEWS
added
removed
shamap.py
25
25
26
26
import bzrlib
27
27
from bzrlib import (
28
registry,
29
28
trace,
30
29
)
31
from bzrlib.transport import (
32
get_transport,
33
)
34
30
35
31
36
32
def get_cache_dir():
46
42
return ret
47
43
48
44
49
def get_remote_cache_transport():
50
return get_transport(get_cache_dir())
51
52
53
45
def check_pysqlite_version(sqlite3):
54
46
"""Check that sqlite library is compatible.
55
47
86
78
class GitShaMap(object):
87
79
"""Git<->Bzr revision id mapping database."""
88
80
81
def add_entry(self, sha, type, type_data):
82
"""Add a new entry to the database.
83
"""
84
raise NotImplementedError(self.add_entry)
85
86
def add_entries(self, entries):
87
"""Add multiple new entries to the database.
88
"""
89
for e in entries:
90
self.add_entry(*e)
91
92
def lookup_tree(self, fileid, revid):
93
"""Lookup the SHA of a git tree."""
94
raise NotImplementedError(self.lookup_tree)
95
96
def lookup_blob(self, fileid, revid):
97
"""Lookup a blob by the fileid it has in a bzr revision."""
98
raise NotImplementedError(self.lookup_blob)
99
89
100
def lookup_git_sha(self, sha):
90
101
"""Lookup a Git sha in the database.
102
91
103
:param sha: Git object sha
92
104
:return: (type, type_data) with type_data:
93
105
revision: revid, tree sha
94
106
"""
95
107
raise NotImplementedError(self.lookup_git_sha)
96
108
97
def lookup_blob_id(self, file_id, revision):
98
"""Retrieve a Git blob SHA by file id.
99
100
:param file_id: File id of the file/symlink
101
:param revision: revision in which the file was last changed.
102
"""
103
raise NotImplementedError(self.lookup_blob_id)
104
105
def lookup_tree_id(self, file_id, revision):
106
"""Retrieve a Git tree SHA by file id.
107
"""
108
raise NotImplementedError(self.lookup_tree_id)
109
110
109
def revids(self):
111
110
"""List the revision ids known."""
112
111
raise NotImplementedError(self.revids)
132
131
"""Abort any pending changes."""
133
132
134
133
135
class ContentCache(object):
136
"""Object that can cache Git objects."""
137
138
def __getitem__(self, sha):
139
"""Retrieve an item, by SHA."""
140
raise NotImplementedError(self.__getitem__)
141
142
143
class BzrGitCacheFormat(object):
144
145
def get_format_string(self):
146
"""Return a single-line unique format string for this cache format."""
147
raise NotImplementedError(self.get_format_string)
148
149
def open(self, transport):
150
"""Open this format on a transport."""
151
raise NotImplementedError(self.open)
152
153
def initialize(self, transport):
154
transport.put_bytes('format', self.get_format_string())
155
156
@classmethod
157
def from_transport(self, transport):
158
"""Open a cache file present on a transport, or initialize one.
159
160
:param transport: Transport to use
161
:return: A BzrGitCache instance
162
"""
163
try:
164
format_name = transport.get_bytes('format')
165
format = formats.get(format_name)
166
except bzrlib.errors.NoSuchFile:
167
format = formats.get('default')
168
format.initialize(transport)
169
return format.open(transport)
170
171
@classmethod
172
def from_repository(cls, repository):
173
"""Open a cache file for a repository.
174
175
This will use the repository's transport to store the cache file, or
176
use the users global cache directory if the repository has no
177
transport associated with it.
178
179
:param repository: Repository to open the cache for
180
:return: A `BzrGitCache`
181
"""
182
repo_transport = getattr(repository, "_transport", None)
183
if repo_transport is not None:
184
# Even if we don't write to this repo, we should be able
185
# to update its cache.
186
repo_transport = remove_readonly_transport_decorator(repo_transport)
187
try:
188
repo_transport.mkdir('git')
189
except bzrlib.errors.FileExists:
190
pass
191
transport = repo_transport.clone('git')
192
else:
193
transport = get_remote_cache_transport()
194
return cls.from_transport(transport)
195
196
197
class CacheUpdater(object):
198
199
def add_object(self, obj, ie):
200
raise NotImplementedError(self.add_object)
201
202
def finish(self):
203
raise NotImplementedError(self.finish)
204
205
206
class BzrGitCache(object):
207
"""Caching backend."""
208
209
def __init__(self, idmap, content_cache, cache_updater_klass):
210
self.idmap = idmap
211
self.content_cache = content_cache
212
self._cache_updater_klass = cache_updater_klass
213
214
def get_updater(self, rev):
215
return self._cache_updater_klass(self, rev)
216
217
218
DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), None, DictCacheUpdater)
219
220
221
class DictCacheUpdater(CacheUpdater):
222
223
def __init__(self, cache, rev):
224
self.cache = cache
225
self.revid = rev.revision_id
226
self.parent_revids = rev.parent_ids
227
self._commit = None
228
self._entries = []
229
230
def add_object(self, obj, ie):
231
if obj.type_name == "commit":
232
self._commit = obj
233
assert ie is None
234
type_data = (self.revid, self._commit.tree)
235
self.cache.idmap._by_revid[self.revid] = obj.id
236
elif obj.type_name in ("blob", "tree"):
237
if ie is not None:
238
if obj.type_name == "blob":
239
revision = ie.revision
240
else:
241
revision = self.revid
242
type_data = (ie.file_id, revision)
243
self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] =\
244
obj.id
245
else:
246
raise AssertionError
247
self.cache.idmap._by_sha[obj.id] = (obj.type_name, type_data)
248
249
def finish(self):
250
if self._commit is None:
251
raise AssertionError("No commit object added")
252
return self._commit
253
254
255
134
class DictGitShaMap(GitShaMap):
256
135
257
136
def __init__(self):
258
self._by_sha = {}
259
self._by_fileid = {}
260
self._by_revid = {}
137
self.dict = {}
261
138
262
def lookup_blob_id(self, fileid, revision):
263
return self._by_fileid[revision][fileid]
139
def add_entry(self, sha, type, type_data):
140
self.dict[sha] = (type, type_data)
264
141
265
142
def lookup_git_sha(self, sha):
266
return self._by_sha[sha]
267
268
def lookup_tree_id(self, fileid, revision):
269
return self._by_fileid[revision][fileid]
270
271
def lookup_commit(self, revid):
272
return self._by_revid[revid]
143
return self.dict[sha]
144
145
def lookup_tree(self, fileid, revid):
146
for k, v in self.dict.iteritems():
147
if v == ("tree", (fileid, revid)):
148
return k
149
raise KeyError((fileid, revid))
150
151
def lookup_blob(self, fileid, revid):
152
for k, v in self.dict.iteritems():
153
if v == ("blob", (fileid, revid)):
154
return k
155
raise KeyError((fileid, revid))
273
156
274
157
def revids(self):
275
for key, (type, type_data) in self._by_sha.iteritems():
158
for key, (type, type_data) in self.dict.iteritems():
276
159
if type == "commit":
277
160
yield type_data[0]
278
161
279
162
def sha1s(self):
280
return self._by_sha.iterkeys()
281
282
283
class SqliteCacheUpdater(CacheUpdater):
284
285
def __init__(self, cache, rev):
286
self.cache = cache
287
self.db = self.cache.idmap.db
288
self.revid = rev.revision_id
289
self._commit = None
290
self._trees = []
291
self._blobs = []
292
293
def add_object(self, obj, ie):
294
if obj.type_name == "commit":
295
self._commit = obj
296
assert ie is None
297
elif obj.type_name == "tree":
298
if ie is not None:
299
self._trees.append((obj.id, ie.file_id, self.revid))
300
elif obj.type_name == "blob":
301
if ie is not None:
302
self._blobs.append((obj.id, ie.file_id, ie.revision))
303
else:
304
raise AssertionError
305
306
def finish(self):
307
if self._commit is None:
308
raise AssertionError("No commit object added")
309
self.db.executemany(
310
"replace into trees (sha1, fileid, revid) values (?, ?, ?)",
311
self._trees)
312
self.db.executemany(
313
"replace into blobs (sha1, fileid, revid) values (?, ?, ?)",
314
self._blobs)
315
self.db.execute(
316
"replace into commits (sha1, revid, tree_sha) values (?, ?, ?)",
317
(self._commit.id, self.revid, self._commit.tree))
318
return self._commit
319
320
321
SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), None, SqliteCacheUpdater)
322
323
324
class SqliteGitCacheFormat(BzrGitCacheFormat):
325
326
def get_format_string(self):
327
return 'bzr-git sha map version 1 using sqlite\n'
328
329
def open(self, transport):
330
try:
331
basepath = transport.local_abspath(".")
332
except bzrlib.errors.NotLocalUrl:
333
basepath = get_cache_dir()
334
return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))
163
return self.dict.iterkeys()
335
164
336
165
337
166
class SqliteGitShaMap(GitShaMap):
361
190
create index if not exists blobs_sha1 on blobs(sha1);
362
191
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
363
192
create table if not exists trees(
364
sha1 text unique not null check(length(sha1) == 40),
193
sha1 text not null check(length(sha1) == 40),
365
194
fileid text not null,
366
195
revid text not null
367
196
);
368
create unique index if not exists trees_sha1 on trees(sha1);
197
create index if not exists trees_sha1 on trees(sha1);
369
198
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
370
199
""")
371
200
372
def __repr__(self):
373
return "%s(%r)" % (self.__class__.__name__, self.path)
374
201
@classmethod
202
def from_repository(cls, repository):
203
try:
204
transport = getattr(repository, "_transport", None)
205
if transport is not None:
206
return cls(os.path.join(transport.local_abspath("."), "git.db"))
207
except bzrlib.errors.NotLocalUrl:
208
pass
209
return cls(os.path.join(get_cache_dir(), "remote.db"))
210
375
211
def lookup_commit(self, revid):
376
212
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
377
213
if row is not None:
381
217
def commit_write_group(self):
382
218
self.db.commit()
383
219
384
def lookup_blob_id(self, fileid, revision):
385
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
386
if row is not None:
387
return row[0]
388
raise KeyError(fileid)
389
390
def lookup_tree_id(self, fileid, revision):
391
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()
392
if row is not None:
393
return row[0]
394
raise KeyError(fileid)
220
def add_entries(self, entries):
221
trees = []
222
blobs = []
223
for sha, type, type_data in entries:
224
assert isinstance(type_data[0], str)
225
assert isinstance(type_data[1], str)
226
entry = (sha, type_data[0], type_data[1])
227
if type == "tree":
228
trees.append(entry)
229
elif type == "blob":
230
blobs.append(entry)
231
else:
232
raise AssertionError
233
if trees:
234
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
235
if blobs:
236
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
237
238
239
def add_entry(self, sha, type, type_data):
240
"""Add a new entry to the database.
241
"""
242
assert isinstance(type_data, tuple)
243
if sha is None:
244
return
245
assert isinstance(sha, str), "type was %r" % sha
246
if type == "commit":
247
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
248
elif type in ("blob", "tree"):
249
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
250
else:
251
raise AssertionError("Unknown type %s" % type)
252
253
def lookup_tree(self, fileid, revid):
254
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid,revid)).fetchone()
255
if row is None:
256
raise KeyError((fileid, revid))
257
return row[0]
258
259
def lookup_blob(self, fileid, revid):
260
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
261
if row is None:
262
raise KeyError((fileid, revid))
263
return row[0]
395
264
396
265
def lookup_git_sha(self, sha):
397
266
"""Lookup a Git sha in the database.
400
269
:return: (type, type_data) with type_data:
401
270
revision: revid, tree sha
402
271
"""
272
def format(type, row):
273
return (type, (row[0], row[1]))
403
274
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
404
275
if row is not None:
405
return ("commit", row)
276
return format("commit", row)
406
277
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
407
278
if row is not None:
408
return ("blob", row)
279
return format("blob", row)
409
280
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
410
281
if row is not None:
411
return ("tree", row)
282
return format("tree", row)
412
283
raise KeyError(sha)
413
284
414
285
def revids(self):
415
286
"""List the revision ids known."""
416
return (row for (row,) in self.db.execute("select revid from commits"))
287
for row in self.db.execute("select revid from commits").fetchall():
288
yield row[0]
417
289
418
290
def sha1s(self):
419
291
"""List the SHA1s."""
420
292
for table in ("blobs", "commits", "trees"):
421
for (sha,) in self.db.execute("select sha1 from %s" % table):
422
yield sha
423
424
425
class TdbCacheUpdater(CacheUpdater):
426
427
def __init__(self, cache, rev):
428
self.cache = cache
429
self.db = cache.idmap.db
430
self.revid = rev.revision_id
431
self.parent_revids = rev.parent_ids
432
self._commit = None
433
self._entries = []
434
435
def add_object(self, obj, ie):
436
sha = obj.sha().digest()
437
if obj.type_name == "commit":
438
self.db["commit\0" + self.revid] = "\0".join((sha, obj.tree))
439
type_data = (self.revid, obj.tree)
440
self._commit = obj
441
assert ie is None
442
elif obj.type_name == "blob":
443
if ie is None:
444
return
445
self.db["\0".join(("blob", ie.file_id, ie.revision))] = sha
446
type_data = (ie.file_id, ie.revision)
447
elif obj.type_name == "tree":
448
if ie is None:
449
return
450
type_data = (ie.file_id, self.revid)
451
else:
452
raise AssertionError
453
self.db["git\0" + sha] = "\0".join((obj.type_name, ) + type_data)
454
455
def finish(self):
456
if self._commit is None:
457
raise AssertionError("No commit object added")
458
return self._commit
459
460
461
TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), None, TdbCacheUpdater)
462
463
class TdbGitCacheFormat(BzrGitCacheFormat):
464
465
def get_format_string(self):
466
return 'bzr-git sha map version 3 using tdb\n'
467
468
def open(self, transport):
469
try:
470
basepath = transport.local_abspath(".")
471
except bzrlib.errors.NotLocalUrl:
472
basepath = get_cache_dir()
473
try:
474
return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))
475
except ImportError:
476
raise ImportError(
477
"Unable to open existing bzr-git cache because 'tdb' is not "
478
"installed.")
293
for row in self.db.execute("select sha1 from %s" % table).fetchall():
294
yield row[0]
295
296
297
TDB_MAP_VERSION = 2
298
TDB_HASH_SIZE = 50000
479
299
480
300
481
301
class TdbGitShaMap(GitShaMap):
489
309
"blob fileid revid" -> "<sha1>"
490
310
"""
491
311
492
TDB_MAP_VERSION = 3
493
TDB_HASH_SIZE = 50000
494
495
312
def __init__(self, path=None):
496
313
import tdb
497
314
self.path = path
499
316
self.db = {}
500
317
else:
501
318
if not mapdbs().has_key(path):
502
mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,
319
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
503
320
os.O_RDWR|os.O_CREAT)
504
321
self.db = mapdbs()[path]
505
322
try:
506
if int(self.db["version"]) not in (2, 3):
323
if int(self.db["version"]) != TDB_MAP_VERSION:
507
324
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
508
self.db["version"], self.TDB_MAP_VERSION)
325
self.db["version"], TDB_MAP_VERSION)
509
326
self.db.clear()
327
self.db["version"] = str(TDB_MAP_VERSION)
510
328
except KeyError:
329
self.db["version"] = str(TDB_MAP_VERSION)
330
331
@classmethod
332
def from_repository(cls, repository):
333
try:
334
transport = getattr(repository, "_transport", None)
335
if transport is not None:
336
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
337
except bzrlib.errors.NotLocalUrl:
511
338
pass
512
self.db["version"] = str(self.TDB_MAP_VERSION)
513
514
def start_write_group(self):
515
"""Start writing changes."""
516
self.db.transaction_start()
517
518
def commit_write_group(self):
519
"""Commit any pending changes."""
520
self.db.transaction_commit()
521
522
def abort_write_group(self):
523
"""Abort any pending changes."""
524
self.db.transaction_cancel()
525
526
def __repr__(self):
527
return "%s(%r)" % (self.__class__.__name__, self.path)
339
return cls(os.path.join(get_cache_dir(), "remote.tdb"))
528
340
529
341
def lookup_commit(self, revid):
530
342
return sha_to_hex(self.db["commit\0" + revid][:20])
531
343
532
def lookup_blob_id(self, fileid, revision):
533
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
534
344
def add_entry(self, hexsha, type, type_data):
345
"""Add a new entry to the database.
346
"""
347
if hexsha is None:
348
sha = ""
349
else:
350
sha = hex_to_sha(hexsha)
351
self.db["git\0" + sha] = "\0".join((type, type_data[0], type_data[1]))
352
if type == "commit":
353
self.db["commit\0" + type_data[0]] = "\0".join((sha, type_data[1]))
354
else:
355
self.db["\0".join((type, type_data[0], type_data[1]))] = sha
356
357
def lookup_tree(self, fileid, revid):
358
sha = self.db["\0".join(("tree", fileid, revid))]
359
if sha == "":
360
return None
361
else:
362
return sha_to_hex(sha)
363
364
def lookup_blob(self, fileid, revid):
365
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
366
535
367
def lookup_git_sha(self, sha):
536
368
"""Lookup a Git sha in the database.
537
369
562
394
for key in self.db.iterkeys():
563
395
if key.startswith("git\0"):
564
396
yield sha_to_hex(key[4:])
565
566
567
formats = registry.Registry()
568
formats.register(TdbGitCacheFormat().get_format_string(),
569
TdbGitCacheFormat())
570
formats.register(SqliteGitCacheFormat().get_format_string(),
571
SqliteGitCacheFormat())
572
try:
573
import tdb
574
except ImportError:
575
formats.register('default', SqliteGitCacheFormat())
576
else:
577
formats.register('default', TdbGitCacheFormat())
578
579
580
def migrate_ancient_formats(repo_transport):
581
# Prefer migrating git.db over git.tdb, since the latter may not
582
# be openable on some platforms.
583
if repo_transport.has("git.db"):
584
SqliteGitCacheFormat().initialize(repo_transport.clone("git"))
585
repo_transport.rename("git.db", "git/idmap.db")
586
elif repo_transport.has("git.tdb"):
587
TdbGitCacheFormat().initialize(repo_transport.clone("git"))
588
repo_transport.rename("git.tdb", "git/idmap.tdb")
589
590
591
def remove_readonly_transport_decorator(transport):
592
if transport.is_readonly():
593
return transport._decorated
594
return transport
595
596
597
def from_repository(repository):
598
"""Open a cache file for a repository.
599
600
If the repository is remote and there is no transport available from it
601
this will use a local file in the users cache directory
602
(typically ~/.cache/bazaar/git/)
603
604
:param repository: A repository object
605
"""
606
repo_transport = getattr(repository, "_transport", None)
607
if repo_transport is not None:
608
# Migrate older cache formats
609
repo_transport = remove_readonly_transport_decorator(repo_transport)
610
try:
611
repo_transport.mkdir("git")
612
except bzrlib.errors.FileExists:
613
pass
614
else:
615
migrate_ancient_formats(repo_transport)
616
return BzrGitCacheFormat.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0