1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Map from Git sha's to Bazaar objects."""
19
from dulwich.objects import (
34
from xdg.BaseDirectory import xdg_cache_home
36
from bzrlib.config import config_dir
37
ret = os.path.join(config_dir(), "git")
39
ret = os.path.join(xdg_cache_home, "bazaar", "git")
40
if not os.path.isdir(ret):
45
def check_pysqlite_version(sqlite3):
46
"""Check that sqlite library is compatible.
49
if (sqlite3.sqlite_version_info[0] < 3 or
50
(sqlite3.sqlite_version_info[0] == 3 and
51
sqlite3.sqlite_version_info[1] < 3)):
52
trace.warning('Needs at least sqlite 3.3.x')
53
raise bzrlib.errors.BzrError("incompatible sqlite library")
58
check_pysqlite_version(sqlite3)
59
except (ImportError, bzrlib.errors.BzrError), e:
60
from pysqlite2 import dbapi2 as sqlite3
61
check_pysqlite_version(sqlite3)
63
trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '
65
raise bzrlib.errors.BzrError("missing sqlite library")
68
_mapdbs = threading.local()
70
"""Get a cache for this thread's db connections."""
73
except AttributeError:
78
class InventorySHAMap(object):
79
"""Maps inventory file ids to Git SHAs."""
81
def lookup_blob(self, file_id, revision):
82
"""Retrieve a Git blob SHA by file id.
84
:param file_id: File id of the file/symlink
85
:param revision: revision in which the file was last changed.
87
raise NotImplementedError(self.lookup_blob)
89
def lookup_tree(self, file_id):
90
"""Retrieve a Git tree SHA by file id.
92
raise NotImplementedError(self.lookup_tree)
95
class GitShaMap(object):
96
"""Git<->Bzr revision id mapping database."""
98
def _add_entry(self, sha, type, type_data):
99
"""Add a new entry to the database.
101
raise NotImplementedError(self._add_entry)
103
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
105
"""Add multiple new entries to the database.
107
for (fileid, kind, hexsha, revision) in entries:
108
self._add_entry(hexsha, kind, (fileid, revision))
109
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
111
def get_inventory_sha_map(self, revid):
112
"""Return the inventory SHA map for a revision.
114
:param revid: Revision to fetch the map for
115
:return: A `InventorySHAMap`
117
raise NotImplementedError(self.get_inventory_sha_map)
119
def lookup_git_sha(self, sha):
120
"""Lookup a Git sha in the database.
121
:param sha: Git object sha
122
:return: (type, type_data) with type_data:
123
revision: revid, tree sha
125
raise NotImplementedError(self.lookup_git_sha)
128
"""List the revision ids known."""
129
raise NotImplementedError(self.revids)
131
def missing_revisions(self, revids):
132
"""Return set of all the revisions that are not present."""
133
present_revids = set(self.revids())
134
if not isinstance(revids, set):
136
return revids - present_revids
139
"""List the SHA1s."""
140
raise NotImplementedError(self.sha1s)
142
def start_write_group(self):
143
"""Start writing changes."""
145
def commit_write_group(self):
146
"""Commit any pending changes."""
148
def abort_write_group(self):
149
"""Abort any pending changes."""
152
class DictGitShaMap(GitShaMap):
158
def _add_entry(self, sha, type, type_data):
159
self._by_sha[sha] = (type, type_data)
160
if type in ("blob", "tree"):
161
self._by_fileid.setdefault(type_data[1], {})[type_data[0]] = sha
163
def get_inventory_sha_map(self, revid):
165
class DictInventorySHAMap(InventorySHAMap):
167
def __init__(self, base, revid):
171
def lookup_blob(self, fileid, revision):
172
return self._base._by_fileid[revision][fileid]
174
def lookup_tree(self, fileid):
175
return self._base._by_fileid[self.revid][fileid]
177
return DictInventorySHAMap(self, revid)
179
def lookup_git_sha(self, sha):
180
return self._by_sha[sha]
183
for key, (type, type_data) in self._by_sha.iteritems():
188
return self._by_sha.iterkeys()
191
class SqliteGitShaMap(GitShaMap):
193
def __init__(self, path=None):
196
self.db = sqlite3.connect(":memory:")
198
if not mapdbs().has_key(path):
199
mapdbs()[path] = sqlite3.connect(path)
200
self.db = mapdbs()[path]
201
self.db.text_factory = str
202
self.db.executescript("""
203
create table if not exists commits(
204
sha1 text not null check(length(sha1) == 40),
206
tree_sha text not null check(length(tree_sha) == 40)
208
create index if not exists commit_sha1 on commits(sha1);
209
create unique index if not exists commit_revid on commits(revid);
210
create table if not exists blobs(
211
sha1 text not null check(length(sha1) == 40),
212
fileid text not null,
215
create index if not exists blobs_sha1 on blobs(sha1);
216
create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);
217
create table if not exists trees(
218
sha1 text unique not null check(length(sha1) == 40),
219
fileid text not null,
222
create unique index if not exists trees_sha1 on trees(sha1);
223
create unique index if not exists trees_fileid_revid on trees(fileid, revid);
227
return "%s(%r)" % (self.__class__.__name__, self.path)
230
def from_repository(cls, repository):
232
transport = getattr(repository, "_transport", None)
233
if transport is not None:
234
return cls(os.path.join(transport.local_abspath("."), "git.db"))
235
except bzrlib.errors.NotLocalUrl:
237
return cls(os.path.join(get_cache_dir(), "remote.db"))
239
def lookup_commit(self, revid):
240
row = self.db.execute("select sha1 from commits where revid = ?", (revid,)).fetchone()
245
def commit_write_group(self):
248
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
252
for (fileid, kind, hexsha, revision) in entries:
256
trees.append((hexsha, fileid, revid))
258
blobs.append((hexsha, fileid, revision))
262
self.db.executemany("replace into trees (sha1, fileid, revid) values (?, ?, ?)", trees)
264
self.db.executemany("replace into blobs (sha1, fileid, revid) values (?, ?, ?)", blobs)
265
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
267
def _add_entry(self, sha, type, type_data):
268
"""Add a new entry to the database.
270
assert isinstance(type_data, tuple)
273
assert isinstance(sha, str), "type was %r" % sha
275
self.db.execute("replace into commits (sha1, revid, tree_sha) values (?, ?, ?)", (sha, type_data[0], type_data[1]))
276
elif type in ("blob", "tree"):
277
self.db.execute("replace into %ss (sha1, fileid, revid) values (?, ?, ?)" % type, (sha, type_data[0], type_data[1]))
279
raise AssertionError("Unknown type %s" % type)
281
def get_inventory_sha_map(self, revid):
282
class SqliteInventorySHAMap(InventorySHAMap):
284
def __init__(self, db, revid):
288
def lookup_blob(self, fileid, revision):
289
row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()
292
raise KeyError(fileid)
294
def lookup_tree(self, fileid):
295
row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, self.revid)).fetchone()
298
raise KeyError(fileid)
300
return SqliteInventorySHAMap(self.db, revid)
302
def lookup_git_sha(self, sha):
303
"""Lookup a Git sha in the database.
305
:param sha: Git object sha
306
:return: (type, type_data) with type_data:
307
revision: revid, tree sha
309
def format(type, row):
310
return (type, (row[0], row[1]))
311
row = self.db.execute("select revid, tree_sha from commits where sha1 = ?", (sha,)).fetchone()
313
return format("commit", row)
314
row = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,)).fetchone()
316
return format("blob", row)
317
row = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,)).fetchone()
319
return format("tree", row)
323
"""List the revision ids known."""
324
return (row for (row,) in self.db.execute("select revid from commits"))
327
"""List the SHA1s."""
328
for table in ("blobs", "commits", "trees"):
330
for (row,) in self.db.execute("select sha1 from %s" % table):
335
TDB_HASH_SIZE = 50000
338
class TdbGitShaMap(GitShaMap):
339
"""SHA Map that uses a TDB database.
343
"git <sha1>" -> "<type> <type-data1> <type-data2>"
344
"commit revid" -> "<sha1> <tree-id>"
345
"tree fileid revid" -> "<sha1>"
346
"blob fileid revid" -> "<sha1>"
349
def __init__(self, path=None):
355
if not mapdbs().has_key(path):
356
mapdbs()[path] = tdb.Tdb(path, TDB_HASH_SIZE, tdb.DEFAULT,
357
os.O_RDWR|os.O_CREAT)
358
self.db = mapdbs()[path]
360
if int(self.db["version"]) not in (2, 3):
361
trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",
362
self.db["version"], TDB_MAP_VERSION)
366
self.db["version"] = str(TDB_MAP_VERSION)
368
def add_entries(self, revid, parent_revids, commit_sha, root_tree_sha,
370
"""Add multiple new entries to the database.
372
self.db.transaction_start()
374
self._add_entry(commit_sha, "commit", (revid, root_tree_sha))
375
for (fileid, kind, hexsha, revision) in entries:
376
self._add_entry(hexsha, kind, (fileid, revision))
378
self.db.transaction_cancel()
381
self.db.transaction_commit()
384
return "%s(%r)" % (self.__class__.__name__, self.path)
387
def from_repository(cls, repository):
389
transport = getattr(repository, "_transport", None)
390
if transport is not None:
391
return cls(os.path.join(transport.local_abspath("."), "git.tdb"))
392
except bzrlib.errors.NotLocalUrl:
394
return cls(os.path.join(get_cache_dir(), "remote.tdb"))
396
def lookup_commit(self, revid):
397
return sha_to_hex(self.db["commit\0" + revid][:20])
399
def _add_entry(self, hexsha, type, type_data):
400
"""Add a new entry to the database.
405
sha = hex_to_sha(hexsha)
406
self.db["git\0" + sha] = "\0".join((type, type_data[0], type_data[1]))
408
self.db["commit\0" + type_data[0]] = "\0".join((sha, type_data[1]))
410
self.db["\0".join(("blob", type_data[0], type_data[1]))] = sha
412
def get_inventory_sha_map(self, revid):
414
class TdbInventorySHAMap(InventorySHAMap):
416
def __init__(self, db, revid):
420
def lookup_blob(self, fileid, revision):
421
return sha_to_hex(self.db["\0".join(("blob", fileid, revision))])
423
return TdbInventorySHAMap(self.db, revid)
425
def lookup_git_sha(self, sha):
426
"""Lookup a Git sha in the database.
428
:param sha: Git object sha
429
:return: (type, type_data) with type_data:
430
revision: revid, tree sha
433
sha = hex_to_sha(sha)
434
data = self.db["git\0" + sha].split("\0")
435
return (data[0], (data[1], data[2]))
437
def missing_revisions(self, revids):
440
if self.db.get("commit\0" + revid) is None:
445
"""List the revision ids known."""
446
for key in self.db.iterkeys():
447
if key.startswith("commit\0"):
451
"""List the SHA1s."""
452
for key in self.db.iterkeys():
453
if key.startswith("git\0"):
454
yield sha_to_hex(key[4:])
457
def from_repository(repository):
459
return TdbGitShaMap.from_repository(repository)
461
return SqliteGitShaMap.from_repository(repository)
added
removed
shamap.py
