1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Map from Git sha's to Bazaar objects."""
19
from dulwich.objects import (
24
from dulwich.object_store import (
35
from bzrlib.revision import (
39
from bzrlib.plugins.git.mapping import (
42
extract_unusual_modes,
46
from bzrlib.plugins.git.shamap import (
47
from_repository as idmap_from_repository,
51
def get_object_store(repo, mapping=None):
52
git = getattr(repo, "_git", None)
54
return git.object_store
55
return BazaarObjectStore(repo, mapping)
58
MAX_INV_CACHE_SIZE = 50 * 1024 * 1024
61
class LRUInventoryCache(object):
63
def __init__(self, repository):
64
def approx_inv_size(inv):
65
# Very rough estimate, 1k per inventory entry
66
return len(inv) * 1024
67
self.repository = repository
68
self._cache = lru_cache.LRUSizeCache(max_size=MAX_INV_CACHE_SIZE,
69
after_cleanup_size=None, compute_size=approx_inv_size)
71
def get_inventory(self, revid):
73
return self._cache[revid]
75
inv = self.repository.get_inventory(revid)
76
self._cache.add(revid, inv)
79
def iter_inventories(self, revids):
80
invs = dict([(k, self._cache.get(k)) for k in revids])
81
for inv in self.repository.iter_inventories(
82
[r for r, v in invs.iteritems() if v is None]):
83
invs[inv.revision_id] = inv
84
self._cache.add(inv.revision_id, inv)
85
return (invs[r] for r in revids)
87
def get_inventories(self, revids):
88
return list(self.iter_inventories(revids))
90
def add(self, revid, inv):
91
self._cache.add(revid, inv)
94
def _check_expected_sha(expected_sha, object):
95
"""Check whether an object matches an expected SHA.
97
:param expected_sha: None or expected SHA as either binary or as hex digest
98
:param object: Object to verify
100
if expected_sha is None:
102
if len(expected_sha) == 40:
103
if expected_sha != object.sha().hexdigest():
104
raise AssertionError("Invalid sha for %r: %s" % (object,
106
elif len(expected_sha) == 20:
107
if expected_sha != object.sha().digest():
108
raise AssertionError("Invalid sha for %r: %s" % (object,
109
sha_to_hex(expected_sha)))
111
raise AssertionError("Unknown length %d for %r" % (len(expected_sha),
115
def _inventory_to_objects(inv, parent_invs, parent_invshamaps,
116
unusual_modes, iter_files_bytes, has_ghost_parents):
117
"""Iterate over the objects that were introduced in a revision.
119
:param inv: Inventory to process
120
:param parent_invs: parent inventory SHA maps
121
:param parent_invshamaps: parent inventory SHA Map
122
:param unusual_modes: Unusual file modes
123
:param iter_files_bytes: Repository.iter_files_bytes-like callback
124
:return: Yields (path, object, ie) entries
129
for path, ie in inv.entries():
130
if ie.kind == "file":
131
if ie.revision != inv.revision_id:
132
for (pinv, pinvshamap) in zip(parent_invs, parent_invshamaps):
134
pie = pinv[ie.file_id]
135
except errors.NoSuchId:
138
if (pie.text_sha1 == ie.text_sha1 and
139
pie.kind == ie.kind):
140
shamap[ie.file_id] = pinvshamap.lookup_blob_id(
141
pie.file_id, pie.revision)
143
if not ie.file_id in shamap:
144
new_blobs.append((path, ie))
145
new_trees[urlutils.dirname(path)] = ie.parent_id
146
elif ie.kind == "symlink":
147
blob = symlink_to_blob(ie)
148
for pinv in parent_invs:
150
pie = pinv[ie.file_id]
151
except errors.NoSuchId:
154
if (ie.kind == pie.kind and
155
ie.symlink_target == pie.symlink_target):
159
new_trees[urlutils.dirname(path)] = ie.parent_id
160
shamap[ie.file_id] = blob.id
161
elif ie.kind == "directory":
162
for (pinv, pinvshamap) in zip(parent_invs, parent_invshamaps):
164
pie = pinv[ie.file_id]
165
except errors.NoSuchId:
168
if (pie.kind == ie.kind and
169
pie.children.keys() == ie.children.keys()):
171
shamap[ie.file_id] = pinvshamap.lookup_tree_id(
173
except (NotImplementedError, KeyError):
178
new_trees[path] = ie.file_id
180
raise AssertionError(ie.kind)
182
for (path, ie), chunks in iter_files_bytes(
183
[(ie.file_id, ie.revision, (path, ie.file_id))
184
for (path, ie) in new_blobs]):
186
obj.data = "".join(chunks)
188
shamap[ie.file_id] = obj.id
190
for fid in unusual_modes:
191
new_trees[inv.id2path(fid)] = inv[fid].parent_id
195
items = new_trees.items()
197
for path, file_id in items:
198
parent_id = inv[file_id].parent_id
199
if parent_id is not None:
200
parent_path = urlutils.dirname(path)
201
new_trees[parent_path] = parent_id
202
trees[path] = file_id
204
def ie_to_hexsha(ie):
206
return shamap[ie.file_id]
208
# Not all cache backends store the tree information,
209
# calculate again from scratch
210
ret = directory_to_tree(ie, ie_to_hexsha, unusual_modes)
215
for path in sorted(trees.keys(), reverse=True):
216
ie = inv[trees[path]]
217
assert ie.kind == "directory"
218
obj = directory_to_tree(ie, ie_to_hexsha, unusual_modes)
221
shamap[ie.file_id] = obj.id
224
class ObjectStoreUpdater(object):
226
def __init__(self, store, rev):
228
self.revid = rev.revision_id
229
self.parent_revids = rev.parent_ids
233
def add_object(self, obj, ie):
234
if obj.type_name == "commit":
237
elif obj.type_name in ("blob", "tree"):
238
if obj.type_name == "blob":
239
revision = ie.revision
241
revision = self.revid
242
self._entries.append((ie.file_id, obj.type_name, obj.id, revision))
247
if self._commit is None:
248
raise AssertionError("No commit object added")
249
self.store._idmap.add_entries(self.revid, self.parent_revids,
250
self._commit.id, self._commit.tree, self._entries)
254
class BazaarObjectStore(BaseObjectStore):
255
"""A Git-style object store backed onto a Bazaar repository."""
257
def __init__(self, repository, mapping=None):
258
self.repository = repository
260
self.mapping = default_mapping
262
self.mapping = mapping
263
self._idmap = idmap_from_repository(repository)
264
self._content_cache = None
265
self.start_write_group = self._idmap.start_write_group
266
self.abort_write_group = self._idmap.abort_write_group
267
self.commit_write_group = self._idmap.commit_write_group
268
self.parent_invs_cache = LRUInventoryCache(self.repository)
270
def _update_sha_map(self, stop_revision=None):
271
graph = self.repository.get_graph()
272
if stop_revision is None:
273
heads = graph.heads(self.repository.all_revision_ids())
275
heads = set([stop_revision])
276
missing_revids = self._idmap.missing_revisions(heads)
278
parents = graph.get_parent_map(heads)
280
for p in parents.values():
281
todo.update([x for x in p if x not in missing_revids])
282
heads = self._idmap.missing_revisions(todo)
283
missing_revids.update(heads)
284
if NULL_REVISION in missing_revids:
285
missing_revids.remove(NULL_REVISION)
286
missing_revids = self.repository.has_revisions(missing_revids)
287
if not missing_revids:
289
self.start_write_group()
291
pb = ui.ui_factory.nested_progress_bar()
293
for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
294
trace.mutter('processing %r', revid)
295
pb.update("updating git map", i, len(missing_revids))
296
self._update_sha_map_revision(revid)
300
self.abort_write_group()
303
self.commit_write_group()
306
self._update_sha_map()
307
return iter(self._idmap.sha1s())
309
def _revision_to_commit(self, rev, tree_sha):
310
def parent_lookup(revid):
312
return self._lookup_revision_sha1(revid)
313
except errors.NoSuchRevision:
314
trace.warning("Ignoring ghost parent %s", revid)
316
return self.mapping.export_commit(rev, tree_sha, parent_lookup)
318
def _revision_to_objects(self, rev, inv):
319
unusual_modes = extract_unusual_modes(rev)
320
present_parents = self.repository.has_revisions(rev.parent_ids)
321
has_ghost_parents = (len(rev.parent_ids) < len(present_parents))
322
parent_invs = self.parent_invs_cache.get_inventories(
323
[p for p in rev.parent_ids if p in present_parents])
324
parent_invshamaps = [self._idmap.get_inventory_sha_map(r) for r in rev.parent_ids if r in present_parents]
326
for path, obj, ie in _inventory_to_objects(inv, parent_invs,
327
parent_invshamaps, unusual_modes,
328
self.repository.iter_files_bytes, has_ghost_parents):
333
if not rev.parent_ids:
336
tree_sha = parent_invshamaps[0][inv.root.file_id]
337
commit_obj = self._revision_to_commit(rev, tree_sha)
339
foreign_revid, mapping = mapping_registry.parse_revision_id(rev.revision_id)
340
except errors.InvalidRevisionId:
343
_check_expected_sha(foreign_revid, commit_obj)
344
yield None, commit_obj, None
346
def _get_updater(self, rev):
347
return ObjectStoreUpdater(self, rev)
349
def _update_sha_map_revision(self, revid):
350
rev = self.repository.get_revision(revid)
351
inv = self.parent_invs_cache.get_inventory(rev.revision_id)
352
updater = self._get_updater(rev)
353
for path, obj, ie in self._revision_to_objects(rev, inv):
355
commit_obj = updater.finish()
358
def _get_blob(self, fileid, revision, expected_sha):
359
"""Return a Git Blob object from a fileid and revision stored in bzr.
361
:param fileid: File id of the text
362
:param revision: Revision of the text
365
chunks = self.repository.iter_files_bytes([(fileid, revision, None)]).next()[1]
366
blob.data = "".join(chunks)
367
if blob.id != expected_sha:
368
# Perhaps it's a symlink ?
369
inv = self.parent_invs_cache.get_inventory(revision)
371
assert entry.kind == 'symlink'
372
blob = symlink_to_blob(entry)
373
_check_expected_sha(expected_sha, blob)
376
def _get_tree(self, fileid, revid, inv, unusual_modes, expected_sha=None):
377
"""Return a Git Tree object from a file id and a revision stored in bzr.
379
:param fileid: fileid in the tree.
380
:param revision: Revision of the tree.
382
invshamap = self._idmap.get_inventory_sha_map(inv.revision_id)
383
def get_ie_sha1(entry):
384
if entry.kind == "directory":
386
return invshamap.lookup_tree_id(entry.file_id)
387
except (NotImplementedError, KeyError):
388
obj = self._get_tree(entry.file_id, revid, inv,
394
elif entry.kind in ("file", "symlink"):
395
return invshamap.lookup_blob_id(entry.file_id, entry.revision)
397
raise AssertionError("unknown entry kind '%s'" % entry.kind)
398
tree = directory_to_tree(inv[fileid], get_ie_sha1, unusual_modes)
399
_check_expected_sha(expected_sha, tree)
402
def get_parents(self, sha):
403
"""Retrieve the parents of a Git commit by SHA1.
405
:param sha: SHA1 of the commit
406
:raises: KeyError, NotCommitError
408
return self[sha].parents
410
def _lookup_revision_sha1(self, revid):
411
"""Return the SHA1 matching a Bazaar revision."""
412
if revid == NULL_REVISION:
415
return self._idmap.lookup_commit(revid)
418
return mapping_registry.parse_revision_id(revid)[0]
419
except errors.InvalidRevisionId:
420
self._update_sha_map(revid)
421
return self._idmap.lookup_commit(revid)
423
def get_raw(self, sha):
424
"""Get the raw representation of a Git object by SHA1.
426
:param sha: SHA1 of the git object
429
return (obj.type, obj.as_raw_string())
431
def __contains__(self, sha):
432
# See if sha is in map
434
(type, type_data) = self._lookup_git_sha(sha)
436
return self.repository.has_revision(type_data[0])
438
return self.repository.texts.has_version(type_data)
440
return self.repository.has_revision(type_data[1])
442
raise AssertionError("Unknown object type '%s'" % type)
446
def _lookup_git_sha(self, sha):
447
# See if sha is in map
449
return self._idmap.lookup_git_sha(sha)
451
# if not, see if there are any unconverted revisions and add them
452
# to the map, search for sha in map again
453
self._update_sha_map()
454
return self._idmap.lookup_git_sha(sha)
456
def __getitem__(self, sha):
457
(type, type_data) = self._lookup_git_sha(sha)
458
# convert object to git object
460
(revid, tree_sha) = type_data
462
rev = self.repository.get_revision(revid)
463
except errors.NoSuchRevision:
464
trace.mutter('entry for %s %s in shamap: %r, but not found in '
465
'repository', type, sha, type_data)
467
commit = self._revision_to_commit(rev, tree_sha)
468
_check_expected_sha(sha, commit)
471
(fileid, revision) = type_data
472
return self._get_blob(fileid, revision, expected_sha=sha)
474
if self._content_cache is not None:
476
return self._content_cache[sha]
479
(fileid, revid) = type_data
481
inv = self.parent_invs_cache.get_inventory(revid)
482
rev = self.repository.get_revision(revid)
483
except errors.NoSuchRevision:
484
trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
486
unusual_modes = extract_unusual_modes(rev)
488
return self._get_tree(fileid, revid, inv, unusual_modes,
490
except errors.NoSuchRevision:
493
raise AssertionError("Unknown object type '%s'" % type)
495
def generate_pack_contents(self, have, want):
496
"""Iterate over the contents of a pack file.
498
:param have: List of SHA1s of objects that should not be sent
499
:param want: List of SHA1s of objects that should be sent
502
for commit_sha in have:
504
(type, (revid, tree_sha)) = self._lookup_git_sha(commit_sha)
508
assert type == "commit"
511
for commit_sha in want:
512
if commit_sha in have:
514
(type, (revid, tree_sha)) = self._lookup_git_sha(commit_sha)
515
assert type == "commit"
519
processed.update(pending)
520
next_map = self.repository.get_parent_map(pending)
522
for item in next_map.iteritems():
524
next_pending.update(p for p in item[1] if p not in processed)
525
pending = next_pending
526
if NULL_REVISION in todo:
527
todo.remove(NULL_REVISION)
528
trace.mutter('sending revisions %r', todo)
530
pb = ui.ui_factory.nested_progress_bar()
532
for i, revid in enumerate(todo):
533
pb.update("generating git objects", i, len(todo))
534
rev = self.repository.get_revision(revid)
535
inv = self.parent_invs_cache.get_inventory(revid)
536
for path, obj, ie in self._revision_to_objects(rev, inv):
537
ret.append((obj, path))
added
removed
object_store.py
