To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to shamap.py
- browse files at revision 0.200.772
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.772
- Committer: Jelmer Vernooij
- Date: 2010-03-28 23:40:53 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100328234053-cfowvcg0spvnqt48
Add docstring.
added
removed
shamap.py
27
27
from bzrlib import (
28
28
trace,
29
29
)
30
from bzrlib.errors import (
31
NoSuchRevision,
32
)
33
30
34
31
35
32
def get_cache_dir():
49
46
"""Check that sqlite library is compatible.
50
47
51
48
"""
52
if (sqlite3.sqlite_version_info[0] < 3 or
53
(sqlite3.sqlite_version_info[0] == 3 and
49
if (sqlite3.sqlite_version_info[0] < 3 or
50
(sqlite3.sqlite_version_info[0] == 3 and
54
51
sqlite3.sqlite_version_info[1] < 3)):
55
warning('Needs at least sqlite 3.3.x')
52
trace.warning('Needs at least sqlite 3.3.x')
56
53
raise bzrlib.errors.BzrError("incompatible sqlite library")
57
54
58
55
try:
59
56
try:
60
57
import sqlite3
61
58
check_pysqlite_version(sqlite3)
62
except (ImportError, bzrlib.errors.BzrError), e:
59
except (ImportError, bzrlib.errors.BzrError), e:
63
60
from pysqlite2 import dbapi2 as sqlite3
64
61
check_pysqlite_version(sqlite3)
65
62
except:
66
warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
63
trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
67
64
'module')
68
65
raise bzrlib.errors.BzrError("missing sqlite library")
69
66
78
75
return _mapdbs.cache
79
76
80
77
78
class InventorySHAMap(object):
79
"""Maps inventory file ids to Git SHAs."""
80
81
def lookup_blob(self, file_id, revision_hint=None):
82
"""Retrieve a Git blob SHA by file id.
83
84
:param file_id: File id of the file/symlink
85
:param revision_hint: Optional revision in which the file was last
86
changed.
87
"""
88
raise NotImplementedError(self.lookup_blob)
89
90
def lookup_tree(self, file_id):
91
"""Retrieve a Git tree SHA by file id.
92
"""
93
raise NotImplementedError(self.lookup_tree)
94
95
81
96
class GitShaMap(object):
82
97
"""Git<->Bzr revision id mapping database."""
83
98
84
def add_entry(self, sha, type, type_data):
99
def _add_entry(self, sha, type, type_data):
85
100
"""Add a new entry to the database.
86
101
"""
87
raise NotImplementedError(self.add_entry)
102
raise NotImplementedError(self._add_entry)
88
103
89
def add_entries(self, entries):
104
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
105
entries):
90
106
"""Add multiple new entries to the database.
91
107
"""
92
for e in entries:
93
self.add_entry(*e)
94
95
def lookup_tree(self, fileid, revid):
96
"""Lookup the SHA of a git tree."""
97
raise NotImplementedError(self.lookup_tree)
98
99
def lookup_blob(self, fileid, revid):
100
"""Lookup a blob by the fileid it has in a bzr revision."""
101
raise NotImplementedError(self.lookup_blob)
108
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
109
for (fileid, kind, hexsha, revision) in entries:
110
self._add_entry(hexsha, kind, (fileid, revision))
111
112
def get_inventory_sha_map(self, revid):
113
"""Return the inventory SHA map for a revision.
114
115
:param revid: Revision to fetch the map for
116
:return: A `InventorySHAMap`
117
"""
118
raise NotImplementedError(self.get_inventory_sha_map)
102
119
103
120
def lookup_git_sha(self, sha):
104
121
"""Lookup a Git sha in the database.
105
106
122
:param sha: Git object sha
107
123
:return: (type, type_data) with type_data:
108
124
revision: revid, tree sha
113
129
"""List the revision ids known."""
114
130
raise NotImplementedError(self.revids)
115
131
116
def sha1s(Self):
132
def missing_revisions(self, revids):
133
"""Return set of all the revisions that are not present."""
134
present_revids = set(self.revids())
135
if not isinstance(revids, set):
136
revids = set(revids)
137
return revids - present_revids
138
139
def sha1s(self):
117
140
"""List the SHA1s."""
118
141
raise NotImplementedError(self.sha1s)
119
142
120
def commit(self):
143
def start_write_group(self):
144
"""Start writing changes."""
145
146
def commit_write_group(self):
121
147
"""Commit any pending changes."""
122
148
149
def abort_write_group(self):
150
"""Abort any pending changes."""
151
123
152
124
153
class DictGitShaMap(GitShaMap):
125
154
126
155
def __init__(self):
127
self.dict = {}
128
129
def add_entry(self, sha, type, type_data):
130
self.dict[sha] = (type, type_data)
156
self._by_sha = {}
157
self._by_fileid = {}
158
159
def _add_entry(self, sha, type, type_data):
160
self._by_sha[sha] = (type, type_data)
161
if type in ("blob", "tree"):
162
self._by_fileid.setdefault(type_data[1], {})[type_data[0]] = sha
163
164
def get_inventory_sha_map(self, revid):
165
class DictInventorySHAMap(InventorySHAMap):
166
167
def __init__(self, base, revid):
168
self._base = base
169
self.revid = revid
170
171
def lookup_blob(self, fileid, revision_hint=None):
172
if revision_hint is not None:
173
revid = revision_hint
174
else:
175
revid = self.revid
176
return self._base._by_fileid[revid][fileid]
177
178
def lookup_tree(self, fileid):
179
return self._base._by_fileid[self.revid][fileid]
180
181
return DictInventorySHAMap(self, revid)
131
182
132
183
def lookup_git_sha(self, sha):
133
return self.dict[sha]
134
135
def lookup_tree(self, fileid, revid):
136
for k, v in self.dict.iteritems():
137
if v == ("tree", (fileid, revid)):
138
return k
139
raise KeyError((fileid, revid))
140
141
def lookup_blob(self, fileid, revid):
142
for k, v in self.dict.iteritems():
143
if v == ("blob", (fileid, revid)):
144
return k
145
raise KeyError((fileid, revid))
184
return self._by_sha[sha]
146
185
147
186
def revids(self):
148
for key, (type, type_data) in self.dict.iteritems():
187
for key, (type, type_data) in self._by_sha.iteritems():
149
188
if type == "commit":
150
189
yield type_data[0]
151
190
152
191
def sha1s(self):
153
return self.dict.iterkeys()
192
return self._by_sha.iterkeys()
154
193
155
194
156
195
class SqliteGitShaMap(GitShaMap):
162
201
else:
163
202
if not mapdbs().has_key(path):
164
203
mapdbs()[path] = sqlite3.connect(path)
165
self.db = mapdbs()[path]
204
self.db = mapdbs()[path]
205
self.db.text_factory = str
166
206
self.db.executescript("""
167
create table if not exists commits(sha1 text, revid text, tree_sha text);
207
create table if not exists commits(
208
sha1 text not null check(length(sha1) == 40),
209
revid text not null,
210
tree_sha text not null check(length(tree_sha) == 40)
211
);
168
212
create index if not exists commit_sha1 on commits(sha1);
169
213
create unique index if not exists commit_revid on commits(revid);
170
create table if not exists blobs(sha1 text, fileid text, revid text);
214
create table if not exists blobs(
215
sha1 text not null check(length(sha1) == 40),
216
fileid text not null,
217
revid text not null
218
);
171
219
create index if not exists blobs_sha1 on blobs(sha1);
172
220
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
173
create table if not exists trees(sha1 text, fileid text, revid text);
174
create index if not exists trees_sha1 on trees(sha1);
221
create table if not exists trees(
222
sha1 text unique not null check(length(sha1) == 40),
223
fileid text not null,
224
revid text not null
225
);
226
create unique index if not exists trees_sha1 on trees(sha1);
175
227
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
176
228
""")
177
229
230
def __repr__(self):
231
return "%s(%r)" % (self.__class__.__name__, self.path)
232
178
233
@classmethod
179
234
def from_repository(cls, repository):
180
235
try:
188
243
def lookup_commit(self, revid):
189
244
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
190
245
if row is not None:
191
return row[0].encode("utf-8")
246
return row[0]
192
247
raise KeyError
193
248
194
def commit(self):
249
def commit_write_group(self):
195
250
self.db.commit()
196
251
197
def add_entries(self, entries):
252
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
253
entries):
254
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
198
255
trees = []
199
256
blobs = []
200
for sha, type, type_data in entries:
201
assert isinstance(type_data[0], str)
202
assert isinstance(type_data[1], str)
203
entry = (sha.decode("utf-8"), type_data[0].decode("utf-8"),
204
type_data[1].decode("utf-8"))
205
if type == "tree":
206
trees.append(entry)
207
elif type == "blob":
208
blobs.append(entry)
257
for (fileid, kind, hexsha, revision) in entries:
258
if kind is None:
259
continue
260
if kind == "tree":
261
trees.append((hexsha, "tree", (fileid, revid)))
262
elif kind == "blob":
263
blobs.append((hexsha, (fileid, revision)))
209
264
else:
210
265
raise AssertionError
211
266
if trees:
214
269
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
215
270
216
271
217
def add_entry(self, sha, type, type_data):
272
def _add_entry(self, sha, type, type_data):
218
273
"""Add a new entry to the database.
219
274
"""
220
275
assert isinstance(type_data, tuple)
276
if sha is None:
277
return
221
278
assert isinstance(sha, str), "type was %r" % sha
222
279
if type == "commit":
223
280
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
226
283
else:
227
284
raise AssertionError("Unknown type %s" % type)
228
285
229
def lookup_tree(self, fileid, revid):
230
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid,revid)).fetchone()
231
if row is None:
232
raise KeyError((fileid, revid))
233
return row[0].encode("utf-8")
234
235
def lookup_blob(self, fileid, revid):
236
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
237
if row is None:
238
raise KeyError((fileid, revid))
239
return row[0].encode("utf-8")
286
def get_inventory_sha_map(self, revid):
287
class SqliteInventorySHAMap(InventorySHAMap):
288
289
def __init__(self, db, revid):
290
self.db = db
291
self.revid = revid
292
293
def lookup_blob(self, fileid, revision_hint=None):
294
if revision_hint is not None:
295
revid = revision_hint
296
else:
297
revid = self.revid
298
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revid)).fetchone()
299
if row is not None:
300
return row[0]
301
raise KeyError(fileid)
302
303
def lookup_tree(self, fileid):
304
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, self.revid)).fetchone()
305
if row is not None:
306
return row[0]
307
raise KeyError(fileid)
308
309
return SqliteInventorySHAMap(self.db, revid)
240
310
241
311
def lookup_git_sha(self, sha):
242
312
"""Lookup a Git sha in the database.
246
316
revision: revid, tree sha
247
317
"""
248
318
def format(type, row):
249
return (type, (row[0].encode("utf-8"), row[1].encode("utf-8")))
319
return (type, (row[0], row[1]))
250
320
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
251
321
if row is not None:
252
322
return format("commit", row)
260
330
261
331
def revids(self):
262
332
"""List the revision ids known."""
263
for row in self.db.execute("select revid from commits").fetchall():
264
yield row[0].encode("utf-8")
333
return (row for (row,) in self.db.execute("select revid from commits"))
265
334
266
335
def sha1s(self):
267
336
"""List the SHA1s."""
268
337
for table in ("blobs", "commits", "trees"):
269
for row in self.db.execute("select sha1 from %s" % table).fetchall():
270
yield row[0].encode("utf-8")
271
272
273
TDB_MAP_VERSION = 2
274
TDB_HASH_SIZE = 10000
338
trace.note(table)
339
for (row,) in self.db.execute("select sha1 from %s" % table):
340
yield row
341
342
343
TDB_MAP_VERSION = 3
344
TDB_HASH_SIZE = 50000
275
345
276
346
277
347
class TdbGitShaMap(GitShaMap):
292
362
self.db = {}
293
363
else:
294
364
if not mapdbs().has_key(path):
295
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
365
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
296
366
os.O_RDWR|os.O_CREAT)
297
self.db = mapdbs()[path]
298
if not "version" in self.db:
299
self.db["version"] = str(TDB_MAP_VERSION)
300
else:
301
if int(self.db["version"]) != TDB_MAP_VERSION:
367
self.db = mapdbs()[path]
368
try:
369
if int(self.db["version"]) not in (2, 3):
302
370
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
303
371
self.db["version"], TDB_MAP_VERSION)
304
372
self.db.clear()
305
self.db["version"] = str(TDB_MAP_VERSION)
373
except KeyError:
374
pass
375
self.db["version"] = str(TDB_MAP_VERSION)
376
377
def __repr__(self):
378
return "%s(%r)" % (self.__class__.__name__, self.path)
306
379
307
380
@classmethod
308
381
def from_repository(cls, repository):
317
390
def lookup_commit(self, revid):
318
391
return sha_to_hex(self.db["commit\0" + revid][:20])
319
392
320
def commit(self):
321
pass
322
323
def add_entry(self, sha, type, type_data):
393
def _add_entry(self, hexsha, type, type_data):
324
394
"""Add a new entry to the database.
325
395
"""
326
self.db["git\0" + hex_to_sha(sha)] = "\0".join((type, type_data[0], type_data[1]))
396
if hexsha is None:
397
sha = ""
398
else:
399
sha = hex_to_sha(hexsha)
400
self.db["git\0" + sha] = "\0".join((type, type_data[0], type_data[1]))
327
401
if type == "commit":
328
self.db["commit\0" + type_data[0]] = "\0".join((hex_to_sha(sha), type_data[1]))
329
else:
330
self.db["\0".join((type, type_data[0], type_data[1]))] = hex_to_sha(sha)
331
332
def lookup_tree(self, fileid, revid):
333
return sha_to_hex(self.db["\0".join(("tree", fileid, revid))])
334
335
def lookup_blob(self, fileid, revid):
336
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
402
self.db["commit\0" + type_data[0]] = "\0".join((sha, type_data[1]))
403
elif type == "blob":
404
self.db["\0".join(("blob", type_data[0], type_data[1]))] = sha
405
406
def get_inventory_sha_map(self, revid):
407
408
class TdbInventorySHAMap(InventorySHAMap):
409
410
def __init__(self, db, revid):
411
self.db = db
412
self.revid = revid
413
414
def lookup_blob(self, fileid, revision_hint=None):
415
if revision_hint is not None:
416
revid = revision_hint
417
else:
418
revid = self.revid
419
return sha_to_hex(self.db["\0".join(("blob", fileid, revid))])
420
421
return TdbInventorySHAMap(self.db, revid)
337
422
338
423
def lookup_git_sha(self, sha):
339
424
"""Lookup a Git sha in the database.
342
427
:return: (type, type_data) with type_data:
343
428
revision: revid, tree sha
344
429
"""
345
data = self.db["git\0" + hex_to_sha(sha)].split("\0")
430
if len(sha) == 40:
431
sha = hex_to_sha(sha)
432
data = self.db["git\0" + sha].split("\0")
346
433
return (data[0], (data[1], data[2]))
347
434
435
def missing_revisions(self, revids):
436
ret = set()
437
for revid in revids:
438
if self.db.get("commit\0" + revid) is None:
439
ret.add(revid)
440
return ret
441
348
442
def revids(self):
349
443
"""List the revision ids known."""
350
444
for key in self.db.iterkeys():
356
450
for key in self.db.iterkeys():
357
451
if key.startswith("git\0"):
358
452
yield sha_to_hex(key[4:])
453
454
455
def from_repository(repository):
456
try:
457
return TdbGitShaMap.from_repository(repository)
458
except ImportError:
459
return SqliteGitShaMap.from_repository(repository)
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0