1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Map from Git sha's to Bazaar objects."""
19
from dulwich.objects import (
24
from dulwich.object_store import (
35
from bzrlib.revision import (
39
from bzrlib.plugins.git.mapping import (
42
extract_unusual_modes,
46
from bzrlib.plugins.git.shamap import (
47
from_repository as idmap_from_repository,
51
def get_object_store(repo, mapping=None):
52
git = getattr(repo, "_git", None)
54
return git.object_store
55
return BazaarObjectStore(repo, mapping)
58
MAX_INV_CACHE_SIZE = 50 * 1024 * 1024
61
class LRUInventoryCache(object):
63
def __init__(self, repository):
64
def approx_inv_size(inv):
65
# Very rough estimate, 1k per inventory entry
66
return len(inv) * 1024
67
self.repository = repository
68
self._cache = lru_cache.LRUSizeCache(max_size=MAX_INV_CACHE_SIZE,
69
after_cleanup_size=None, compute_size=approx_inv_size)
71
def get_inventory(self, revid):
73
return self._cache[revid]
75
inv = self.repository.get_inventory(revid)
76
self._cache.add(revid, inv)
79
def iter_inventories(self, revids):
80
invs = dict([(k, self._cache.get(k)) for k in revids])
81
for inv in self.repository.iter_inventories(
82
[r for r, v in invs.iteritems() if v is None]):
83
invs[inv.revision_id] = inv
84
self._cache.add(inv.revision_id, inv)
85
return (invs[r] for r in revids)
87
def get_inventories(self, revids):
88
return list(self.iter_inventories(revids))
90
def add(self, revid, inv):
91
self._cache.add(revid, inv)
94
def _check_expected_sha(expected_sha, object):
95
"""Check whether an object matches an expected SHA.
97
:param expected_sha: None or expected SHA as either binary or as hex digest
98
:param object: Object to verify
100
if expected_sha is None:
102
if len(expected_sha) == 40:
103
if expected_sha != object.sha().hexdigest():
104
raise AssertionError("Invalid sha for %r: %s" % (object,
106
elif len(expected_sha) == 20:
107
if expected_sha != object.sha().digest():
108
raise AssertionError("Invalid sha for %r: %s" % (object,
109
sha_to_hex(expected_sha)))
111
raise AssertionError("Unknown length %d for %r" % (len(expected_sha),
115
def _inventory_to_objects(inv, parent_invs, parent_invshamaps,
116
unusual_modes, iter_files_bytes, has_ghost_parents):
117
"""Iterate over the objects that were introduced in a revision.
119
:param inv: Inventory to process
120
:param parent_invs: parent inventory SHA maps
121
:param parent_invshamaps: parent inventory SHA Map
122
:param unusual_modes: Unusual file modes
123
:param iter_files_bytes: Repository.iter_files_bytes-like callback
124
:return: Yields (path, object) entries
129
for path, ie in inv.entries():
130
if ie.kind == "file":
131
if ie.revision != inv.revision_id:
132
for (pinv, pinvshamap) in zip(parent_invs, parent_invshamaps):
134
pie = pinv[ie.file_id]
135
except errors.NoSuchId:
138
if (pie.text_sha1 == ie.text_sha1 and
139
pie.kind == ie.kind):
140
shamap[ie.file_id] = pinvshamap.lookup_blob(
141
pie.file_id, pie.revision)
143
if not ie.file_id in shamap:
144
new_blobs.append((path, ie))
145
new_trees[urlutils.dirname(path)] = ie.parent_id
146
elif ie.kind == "symlink":
147
blob = symlink_to_blob(ie)
148
for pinv in parent_invs:
150
pie = pinv[ie.file_id]
151
except errors.NoSuchId:
154
if (ie.kind == pie.kind and
155
ie.symlink_target == pie.symlink_target):
159
new_trees[urlutils.dirname(path)] = ie.parent_id
160
shamap[ie.file_id] = blob.id
161
elif ie.kind == "directory":
162
for (pinv, pinvshamap) in zip(parent_invs, parent_invshamaps):
164
pie = pinv[ie.file_id]
165
except errors.NoSuchId:
168
if (pie.kind == ie.kind and
169
pie.children.keys() == ie.children.keys()):
171
shamap[ie.file_id] = pinvshamap.lookup_tree(
173
except NotImplementedError:
178
new_trees[path] = ie.file_id
180
raise AssertionError(ie.kind)
182
for (path, fid), chunks in iter_files_bytes(
183
[(ie.file_id, ie.revision, (path, ie.file_id))
184
for (path, ie) in new_blobs]):
186
obj.data = "".join(chunks)
190
for fid in unusual_modes:
191
new_trees[inv.id2path(fid)] = inv[fid].parent_id
195
items = new_trees.items()
197
for path, file_id in items:
198
parent_id = inv[file_id].parent_id
199
if parent_id is not None:
200
parent_path = urlutils.dirname(path)
201
new_trees[parent_path] = parent_id
202
trees[path] = file_id
204
for path in sorted(trees.keys(), reverse=True):
205
ie = inv[trees[path]]
206
assert ie.kind == "directory"
207
obj = directory_to_tree(ie,
208
lambda ie: shamap[ie.file_id], unusual_modes)
211
shamap[ie.file_id] = obj.id
214
class BazaarObjectStore(BaseObjectStore):
215
"""A Git-style object store backed onto a Bazaar repository."""
217
def __init__(self, repository, mapping=None):
218
self.repository = repository
220
self.mapping = default_mapping
222
self.mapping = mapping
223
self._idmap = idmap_from_repository(repository)
224
self.start_write_group = self._idmap.start_write_group
225
self.abort_write_group = self._idmap.abort_write_group
226
self.commit_write_group = self._idmap.commit_write_group
227
self.parent_invs_cache = LRUInventoryCache(self.repository)
229
def _update_sha_map(self, stop_revision=None):
230
graph = self.repository.get_graph()
231
if stop_revision is None:
232
heads = graph.heads(self.repository.all_revision_ids())
234
heads = set([stop_revision])
235
missing_revids = self._idmap.missing_revisions(heads)
237
parents = graph.get_parent_map(heads)
239
for p in parents.values():
240
todo.update([x for x in p if x not in missing_revids])
241
heads = self._idmap.missing_revisions(todo)
242
missing_revids.update(heads)
243
if NULL_REVISION in missing_revids:
244
missing_revids.remove(NULL_REVISION)
245
missing_revids = self.repository.has_revisions(missing_revids)
246
if not missing_revids:
248
self.start_write_group()
250
pb = ui.ui_factory.nested_progress_bar()
252
for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
253
trace.mutter('processing %r', revid)
254
pb.update("updating git map", i, len(missing_revids))
255
self._update_sha_map_revision(revid)
259
self.abort_write_group()
262
self.commit_write_group()
265
self._update_sha_map()
266
return iter(self._idmap.sha1s())
268
def _revision_to_commit(self, rev, tree_sha):
269
def parent_lookup(revid):
271
return self._lookup_revision_sha1(revid)
272
except errors.NoSuchRevision:
273
trace.warning("Ignoring ghost parent %s", revid)
275
return self.mapping.export_commit(rev, tree_sha, parent_lookup)
277
def _revision_to_objects(self, rev, inv):
278
unusual_modes = extract_unusual_modes(rev)
279
present_parents = self.repository.has_revisions(rev.parent_ids)
280
has_ghost_parents = (len(rev.parent_ids) < len(present_parents))
281
parent_invs = self.parent_invs_cache.get_inventories(
282
[p for p in rev.parent_ids if p in present_parents])
283
parent_invshamaps = [self._idmap.get_inventory_sha_map(r) for r in rev.parent_ids if r in present_parents]
285
for path, obj in _inventory_to_objects(inv, parent_invs,
286
parent_invshamaps, unusual_modes,
287
self.repository.iter_files_bytes, has_ghost_parents):
292
if not rev.parent_ids:
295
tree_sha = parent_invshamaps[0][inv.root.file_id]
296
commit_obj = self._revision_to_commit(rev, tree_sha)
298
foreign_revid, mapping = mapping_registry.parse_revision_id(rev.revision_id)
299
except errors.InvalidRevisionId:
302
_check_expected_sha(foreign_revid, commit_obj)
303
yield None, commit_obj
305
def _update_sha_map_revision(self, revid):
306
rev = self.repository.get_revision(revid)
307
inv = self.parent_invs_cache.get_inventory(rev.revision_id)
310
for path, obj in self._revision_to_objects(rev, inv):
311
if obj._type == "commit":
313
elif obj._type in ("blob", "tree"):
314
file_id = inv.path2id(path)
316
if obj._type == "blob":
317
revision = ie.revision
320
entries.append((file_id, obj._type, obj.id, revision))
323
self._idmap.add_entries(revid, rev.parent_ids, commit_obj.id,
324
commit_obj.tree, entries)
327
def _get_blob(self, fileid, revision, expected_sha):
328
"""Return a Git Blob object from a fileid and revision stored in bzr.
330
:param fileid: File id of the text
331
:param revision: Revision of the text
334
chunks = self.repository.iter_files_bytes([(fileid, revision, None)]).next()[1]
335
blob.data = "".join(chunks)
336
if blob.id != expected_sha:
337
# Perhaps it's a symlink ?
338
inv = self.parent_invs_cache.get_inventory(revision)
340
assert entry.kind == 'symlink'
341
blob = symlink_to_blob(entry)
342
_check_expected_sha(expected_sha, blob)
345
def _get_tree(self, fileid, revid, inv, unusual_modes, expected_sha=None):
346
"""Return a Git Tree object from a file id and a revision stored in bzr.
348
:param fileid: fileid in the tree.
349
:param revision: Revision of the tree.
351
invshamap = self._idmap.get_inventory_sha_map(inv.revision_id)
352
def get_ie_sha1(entry):
353
if entry.kind == "directory":
354
return invshamap.lookup_tree(entry.file_id)
355
elif entry.kind in ("file", "symlink"):
356
return invshamap.lookup_blob(entry.file_id, entry.revision)
358
raise AssertionError("unknown entry kind '%s'" % entry.kind)
359
tree = directory_to_tree(inv[fileid], get_ie_sha1, unusual_modes)
360
_check_expected_sha(expected_sha, tree)
363
def get_parents(self, sha):
364
"""Retrieve the parents of a Git commit by SHA1.
366
:param sha: SHA1 of the commit
367
:raises: KeyError, NotCommitError
369
return self[sha].parents
371
def _lookup_revision_sha1(self, revid):
372
"""Return the SHA1 matching a Bazaar revision."""
373
if revid == NULL_REVISION:
376
return self._idmap.lookup_commit(revid)
379
return mapping_registry.parse_revision_id(revid)[0]
380
except errors.InvalidRevisionId:
381
self._update_sha_map(revid)
382
return self._idmap.lookup_commit(revid)
384
def get_raw(self, sha):
385
"""Get the raw representation of a Git object by SHA1.
387
:param sha: SHA1 of the git object
390
return (obj.type, obj.as_raw_string())
392
def __contains__(self, sha):
393
# See if sha is in map
395
(type, type_data) = self._lookup_git_sha(sha)
397
return self.repository.has_revision(type_data[0])
399
return self.repository.texts.has_version(type_data)
401
return self.repository.has_revision(type_data[1])
403
raise AssertionError("Unknown object type '%s'" % type)
407
def _lookup_git_sha(self, sha):
408
# See if sha is in map
410
return self._idmap.lookup_git_sha(sha)
412
# if not, see if there are any unconverted revisions and add them
413
# to the map, search for sha in map again
414
self._update_sha_map()
415
return self._idmap.lookup_git_sha(sha)
417
def __getitem__(self, sha):
418
(type, type_data) = self._lookup_git_sha(sha)
419
# convert object to git object
421
(revid, tree_sha) = type_data
423
rev = self.repository.get_revision(revid)
424
except errors.NoSuchRevision:
425
trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
427
commit = self._revision_to_commit(rev, tree_sha)
428
_check_expected_sha(sha, commit)
431
(fileid, revision) = type_data
432
return self._get_blob(fileid, revision, expected_sha=sha)
434
(fileid, revid) = type_data
436
inv = self.parent_invs_cache.get_inventory(revid)
437
rev = self.repository.get_revision(revid)
438
except errors.NoSuchRevision:
439
trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
441
unusual_modes = extract_unusual_modes(rev)
443
return self._get_tree(fileid, revid, inv,
444
unusual_modes, expected_sha=sha)
445
except errors.NoSuchRevision:
448
raise AssertionError("Unknown object type '%s'" % type)
450
def generate_pack_contents(self, have, want):
451
"""Iterate over the contents of a pack file.
453
:param have: List of SHA1s of objects that should not be sent
454
:param want: List of SHA1s of objects that should be sent
457
for commit_sha in have:
459
(type, (revid, tree_sha)) = self._lookup_git_sha(commit_sha)
463
assert type == "commit"
466
for commit_sha in want:
467
if commit_sha in have:
469
(type, (revid, tree_sha)) = self._lookup_git_sha(commit_sha)
470
assert type == "commit"
474
processed.update(pending)
475
next_map = self.repository.get_parent_map(pending)
477
for item in next_map.iteritems():
479
next_pending.update(p for p in item[1] if p not in processed)
480
pending = next_pending
481
if NULL_REVISION in todo:
482
todo.remove(NULL_REVISION)
483
trace.mutter('sending revisions %r', todo)
485
pb = ui.ui_factory.nested_progress_bar()
487
for i, revid in enumerate(todo):
488
pb.update("generating git objects", i, len(todo))
489
rev = self.repository.get_revision(revid)
490
inv = self.parent_invs_cache.get_inventory(revid)
491
for path, obj in self._revision_to_objects(rev, inv):
492
ret.append((obj, path))
added
removed
object_store.py
