13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from bzrlib import osutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
23
from bzrlib.plugins.git import git
24
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
25
from bzrlib.plugins.git.remote import RemoteGitRepository
27
from dulwich.objects import Commit
29
from cStringIO import StringIO
32
class BzrFetchGraphWalker(object):
34
def __init__(self, repository, mapping):
35
self.repository = repository
36
self.mapping = mapping
38
self.heads = set(repository.all_revision_ids())
42
revid = self.mapping.revision_id_foreign_to_bzr(sha)
45
def remove(self, revid):
48
self.heads.remove(revid)
49
if revid in self.parents:
50
for p in self.parents[revid]:
55
ret = self.heads.pop()
56
ps = self.repository.get_parent_map([ret])[ret]
57
self.parents[ret] = ps
58
self.heads.update([p for p in ps if not p in self.done])
61
return self.mapping.revision_id_bzr_to_foreign(ret)
62
except InvalidRevisionId:
67
def import_git_blob(repo, mapping, path, blob):
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Fetching from git into bzr."""
19
from __future__ import absolute_import
21
from dulwich.objects import (
29
from dulwich.object_store import (
32
from dulwich.walk import Walker
43
from ...errors import (
46
from ...bzr.inventory import (
52
from ...repository import (
55
from ...revision import (
58
from ...bzr.inventorytree import InventoryRevisionTree
59
from ...testament import (
62
from ...tsort import (
65
from ...bzr.versionedfile import (
66
ChunkedContentFactory,
69
from .mapping import (
75
from .object_store import (
85
from .repository import (
92
def import_git_blob(texts, mapping, path, name, hexshas,
93
base_bzr_tree, parent_id, revision_id,
94
parent_bzr_trees, lookup_object, modes, store_updater,
68
96
"""Import a git blob object into a bzr repository.
70
:param repo: bzr repository
98
:param texts: VersionedFiles to add to
71
99
:param path: Path in the tree
72
100
:param blob: A git blob
74
file_id = mapping.generate_file_id(path)
75
repo.texts.add_lines((file_id, blob.id),
77
osutils.split_lines(blob.data))
78
inv.add_path(path, "file", file_id)
81
def import_git_tree(repo, mapping, path, tree, inv, lookup_object):
101
:return: Inventory delta for this file
103
(base_mode, mode) = modes
104
(base_hexsha, hexsha) = hexshas
105
if mapping.is_special_file(path):
107
if base_hexsha == hexsha and base_mode == mode:
108
# If nothing has changed since the base revision, we're done
110
file_id = lookup_file_id(path)
111
if stat.S_ISLNK(mode):
115
ie = cls(file_id, name.decode("utf-8"), parent_id)
116
if ie.kind == "file":
117
ie.executable = mode_is_executable(mode)
118
if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):
119
base_exec = base_bzr_tree.is_executable(path)
120
if ie.kind == "symlink":
121
ie.symlink_target = base_bzr_tree.get_symlink_target(path)
123
ie.text_size = base_bzr_tree.get_file_size(path)
124
ie.text_sha1 = base_bzr_tree.get_file_sha1(path)
125
if ie.kind == "symlink" or ie.executable == base_exec:
126
ie.revision = base_bzr_tree.get_file_revision(path)
128
blob = lookup_object(hexsha)
130
blob = lookup_object(hexsha)
131
if ie.kind == "symlink":
133
ie.symlink_target = blob.data.decode("utf-8")
135
ie.text_size = sum(map(len, blob.chunked))
136
ie.text_sha1 = osutils.sha_strings(blob.chunked)
137
# Check what revision we should store
139
for ptree in parent_bzr_trees:
141
ppath = ptree.id2path(file_id)
142
except errors.NoSuchId:
144
pkind = ptree.kind(ppath, file_id)
145
if (pkind == ie.kind and
146
((pkind == "symlink" and ptree.get_symlink_target(ppath, file_id) == ie.symlink_target) or
147
(pkind == "file" and ptree.get_file_sha1(ppath, file_id) == ie.text_sha1 and
148
ptree.is_executable(ppath, file_id) == ie.executable))):
149
# found a revision in one of the parents to use
150
ie.revision = ptree.get_file_revision(ppath, file_id)
152
parent_key = (file_id, ptree.get_file_revision(ppath, file_id))
153
if not parent_key in parent_keys:
154
parent_keys.append(parent_key)
155
if ie.revision is None:
156
# Need to store a new revision
157
ie.revision = revision_id
158
if ie.revision is None:
159
raise ValueError("no file revision set")
160
if ie.kind == 'symlink':
163
chunks = blob.chunked
164
texts.insert_record_stream([
165
ChunkedContentFactory((file_id, ie.revision),
166
tuple(parent_keys), ie.text_sha1, chunks)])
168
if base_hexsha is not None:
169
old_path = path.decode("utf-8") # Renames are not supported yet
170
if stat.S_ISDIR(base_mode):
171
invdelta.extend(remove_disappeared_children(base_bzr_tree, old_path,
172
lookup_object(base_hexsha), [], lookup_object))
175
new_path = path.decode("utf-8")
176
invdelta.append((old_path, new_path, file_id, ie))
177
if base_hexsha != hexsha:
178
store_updater.add_object(blob, (ie.file_id, ie.revision), path)
182
class SubmodulesRequireSubtrees(BzrError):
183
_fmt = ("The repository you are fetching from contains submodules, "
184
"which require a Bazaar format that supports tree references.")
188
def import_git_submodule(texts, mapping, path, name, hexshas,
189
base_bzr_tree, parent_id, revision_id, parent_bzr_trees, lookup_object,
190
modes, store_updater, lookup_file_id):
191
"""Import a git submodule."""
192
(base_hexsha, hexsha) = hexshas
193
(base_mode, mode) = modes
194
if base_hexsha == hexsha and base_mode == mode:
196
file_id = lookup_file_id(path)
198
ie = TreeReference(file_id, name.decode("utf-8"), parent_id)
199
ie.revision = revision_id
200
if base_hexsha is not None:
201
old_path = path.decode("utf-8") # Renames are not supported yet
202
if stat.S_ISDIR(base_mode):
203
invdelta.extend(remove_disappeared_children(base_bzr_tree, old_path,
204
lookup_object(base_hexsha), [], lookup_object))
207
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
208
texts.insert_record_stream([
209
ChunkedContentFactory((file_id, ie.revision), (), None, [])])
210
invdelta.append((old_path, path, file_id, ie))
214
def remove_disappeared_children(base_bzr_tree, path, base_tree, existing_children,
216
"""Generate an inventory delta for removed children.
218
:param base_bzr_tree: Base bzr tree against which to generate the
220
:param path: Path to process (unicode)
221
:param base_tree: Git Tree base object
222
:param existing_children: Children that still exist
223
:param lookup_object: Lookup a git object by its SHA1
224
:return: Inventory delta, as list
226
if type(path) is not unicode:
227
raise TypeError(path)
229
for name, mode, hexsha in base_tree.iteritems():
230
if name in existing_children:
232
c_path = posixpath.join(path, name.decode("utf-8"))
233
file_id = base_bzr_tree.path2id(c_path)
235
raise TypeError(file_id)
236
ret.append((c_path, None, file_id, None))
237
if stat.S_ISDIR(mode):
238
ret.extend(remove_disappeared_children(
239
base_bzr_tree, c_path, lookup_object(hexsha), [], lookup_object))
243
def import_git_tree(texts, mapping, path, name, hexshas,
244
base_bzr_tree, parent_id, revision_id, parent_bzr_trees,
245
lookup_object, modes, store_updater,
246
lookup_file_id, allow_submodules=False):
82
247
"""Import a git tree object into a bzr repository.
84
:param repo: A Bzr repository object
85
:param path: Path in the tree
249
:param texts: VersionedFiles object to add to
250
:param path: Path in the tree (str)
251
:param name: Name of the tree (str)
86
252
:param tree: A git tree object
87
:param inv: Inventory object
253
:param base_bzr_tree: Base inventory against which to return inventory delta
254
:return: Inventory delta for this subtree
89
file_id = mapping.generate_file_id(path)
90
repo.texts.add_lines((file_id, tree.id),
93
inv.add_path(path, "directory", file_id)
94
for mode, name, hexsha in tree.entries():
95
entry_kind = (mode & 0700000) / 0100000
96
basename = name.decode("utf-8")
100
child_path = urlutils.join(path, name)
102
import_git_tree(repo, mapping, child_path, lookup_object, inv)
103
elif entry_kind == 1:
104
import_git_blob(repo, mapping, child_path, lookup_object, inv)
106
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
109
def import_git_objects(repo, mapping, object_iter):
256
(base_hexsha, hexsha) = hexshas
257
(base_mode, mode) = modes
258
if type(path) is not str:
259
raise TypeError(path)
260
if type(name) is not str:
261
raise TypeError(name)
262
if base_hexsha == hexsha and base_mode == mode:
263
# If nothing has changed since the base revision, we're done
266
file_id = lookup_file_id(path)
267
# We just have to hope this is indeed utf-8:
268
ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)
269
tree = lookup_object(hexsha)
270
if base_hexsha is None:
272
old_path = None # Newly appeared here
274
base_tree = lookup_object(base_hexsha)
275
old_path = path.decode("utf-8") # Renames aren't supported yet
276
new_path = path.decode("utf-8")
277
if base_tree is None or type(base_tree) is not Tree:
278
ie.revision = revision_id
279
invdelta.append((old_path, new_path, ie.file_id, ie))
280
texts.insert_record_stream([
281
ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
282
# Remember for next time
283
existing_children = set()
285
for name, child_mode, child_hexsha in tree.iteritems():
286
existing_children.add(name)
287
child_path = posixpath.join(path, name)
288
if type(base_tree) is Tree:
290
child_base_mode, child_base_hexsha = base_tree[name]
292
child_base_hexsha = None
295
child_base_hexsha = None
297
if stat.S_ISDIR(child_mode):
298
subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
299
child_path, name, (child_base_hexsha, child_hexsha),
300
base_bzr_tree, file_id, revision_id, parent_bzr_trees,
301
lookup_object, (child_base_mode, child_mode), store_updater,
302
lookup_file_id, allow_submodules=allow_submodules)
303
elif S_ISGITLINK(child_mode): # submodule
304
if not allow_submodules:
305
raise SubmodulesRequireSubtrees()
306
subinvdelta, grandchildmodes = import_git_submodule(texts, mapping,
307
child_path, name, (child_base_hexsha, child_hexsha),
308
base_bzr_tree, file_id, revision_id, parent_bzr_trees,
309
lookup_object, (child_base_mode, child_mode), store_updater,
312
if not mapping.is_special_file(name):
313
subinvdelta = import_git_blob(texts, mapping, child_path, name,
314
(child_base_hexsha, child_hexsha), base_bzr_tree, file_id,
315
revision_id, parent_bzr_trees, lookup_object,
316
(child_base_mode, child_mode), store_updater, lookup_file_id)
320
child_modes.update(grandchildmodes)
321
invdelta.extend(subinvdelta)
322
if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
323
stat.S_IFLNK, DEFAULT_FILE_MODE|0o111,
325
child_modes[child_path] = child_mode
326
# Remove any children that have disappeared
327
if base_tree is not None and type(base_tree) is Tree:
328
invdelta.extend(remove_disappeared_children(base_bzr_tree, old_path,
329
base_tree, existing_children, lookup_object))
330
store_updater.add_object(tree, (file_id, revision_id), path)
331
return invdelta, child_modes
334
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
335
o, rev, ret_tree, parent_trees, mapping, unusual_modes, verifiers):
336
new_unusual_modes = mapping.export_unusual_file_modes(rev)
337
if new_unusual_modes != unusual_modes:
338
raise AssertionError("unusual modes don't match: %r != %r" % (
339
unusual_modes, new_unusual_modes))
340
# Verify that we can reconstruct the commit properly
341
rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True,
344
raise AssertionError("Reconstructed commit differs: %r != %r" % (
348
for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,
349
target_git_object_retriever._cache.idmap, unusual_modes,
350
mapping.BZR_DUMMY_FILE):
351
old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]
353
if obj.id != old_obj_id:
354
diff.append((path, lookup_object(old_obj_id), obj))
355
for (path, old_obj, new_obj) in diff:
356
while (old_obj.type_name == "tree" and
357
new_obj.type_name == "tree" and
358
sorted(old_obj) == sorted(new_obj)):
360
if old_obj[name][0] != new_obj[name][0]:
361
raise AssertionError("Modes for %s differ: %o != %o" %
362
(path, old_obj[name][0], new_obj[name][0]))
363
if old_obj[name][1] != new_obj[name][1]:
364
# Found a differing child, delve deeper
365
path = posixpath.join(path, name)
366
old_obj = lookup_object(old_obj[name][1])
367
new_obj = new_objs[path]
369
raise AssertionError("objects differ for %s: %r != %r" % (path,
373
def ensure_inventories_in_repo(repo, trees):
374
real_inv_vf = repo.inventories.without_fallbacks()
376
revid = t.get_revision_id()
377
if not real_inv_vf.get_parent_map([(revid, )]):
378
repo.add_inventory(revid, t.inventory, t.get_parent_ids())
381
def import_git_commit(repo, mapping, head, lookup_object,
382
target_git_object_retriever, trees_cache):
383
o = lookup_object(head)
384
# Note that this uses mapping.revision_id_foreign_to_bzr. If the parents
385
# were bzr roundtripped revisions they would be specified in the
387
rev, roundtrip_revid, verifiers = mapping.import_commit(
388
o, mapping.revision_id_foreign_to_bzr)
389
if roundtrip_revid is not None:
390
original_revid = rev.revision_id
391
rev.revision_id = roundtrip_revid
392
# We have to do this here, since we have to walk the tree and
393
# we need to make sure to import the blobs / trees with the right
394
# path; this may involve adding them more than once.
395
parent_trees = trees_cache.revision_trees(rev.parent_ids)
396
ensure_inventories_in_repo(repo, parent_trees)
397
if parent_trees == []:
398
base_bzr_tree = trees_cache.revision_tree(NULL_REVISION)
402
base_bzr_tree = parent_trees[0]
403
base_tree = lookup_object(o.parents[0]).tree
404
base_mode = stat.S_IFDIR
405
store_updater = target_git_object_retriever._get_updater(rev)
406
tree_supplement = mapping.get_fileid_map(lookup_object, o.tree)
407
inv_delta, unusual_modes = import_git_tree(repo.texts,
408
mapping, "", "", (base_tree, o.tree), base_bzr_tree,
409
None, rev.revision_id, parent_trees,
410
lookup_object, (base_mode, stat.S_IFDIR), store_updater,
411
tree_supplement.lookup_file_id,
412
allow_submodules=getattr(repo._format, "supports_tree_reference",
414
if unusual_modes != {}:
415
for path, mode in unusual_modes.iteritems():
416
warn_unusual_mode(rev.foreign_revid, path, mode)
417
mapping.import_unusual_file_modes(rev, unusual_modes)
419
basis_id = rev.parent_ids[0]
421
basis_id = NULL_REVISION
422
base_bzr_inventory = None
425
base_bzr_inventory = base_bzr_tree.root_inventory
426
except AttributeError: # bzr < 2.6
427
base_bzr_inventory = base_bzr_tree.inventory
428
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
429
inv_delta, rev.revision_id, rev.parent_ids,
431
ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
433
if verifiers and roundtrip_revid is not None:
434
testament = StrictTestament3(rev, ret_tree)
435
calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
436
if calculated_verifiers != verifiers:
437
trace.mutter("Testament SHA1 %r for %r did not match %r.",
438
calculated_verifiers["testament3-sha1"],
439
rev.revision_id, verifiers["testament3-sha1"])
440
rev.revision_id = original_revid
441
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
442
inv_delta, rev.revision_id, rev.parent_ids, base_bzr_tree)
443
ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
445
calculated_verifiers = {}
446
store_updater.add_object(o, calculated_verifiers, None)
447
store_updater.finish()
448
trees_cache.add(ret_tree)
449
repo.add_revision(rev.revision_id, rev)
450
if "verify" in debug.debug_flags:
451
verify_commit_reconstruction(target_git_object_retriever,
452
lookup_object, o, rev, ret_tree, parent_trees, mapping,
453
unusual_modes, verifiers)
456
def import_git_objects(repo, mapping, object_iter,
457
target_git_object_retriever, heads, pb=None, limit=None):
110
458
"""Import a set of git objects into a bzr repository.
112
:param repo: Bazaar repository
460
:param repo: Target Bazaar repository
113
461
:param mapping: Mapping to use
114
462
:param object_iter: Iterator over Git objects.
463
:return: Tuple with pack hints and last imported revision id
116
# TODO: a more (memory-)efficient implementation of this
118
for o in object_iter:
465
def lookup_object(sha):
467
return object_iter[sha]
469
return target_git_object_retriever[sha]
472
heads = list(set(heads))
473
trees_cache = LRUTreeCache(repo)
121
474
# Find and convert commit objects
122
for o in objects.iterkeys():
477
pb.update("finding revisions to fetch", len(graph), None)
481
if type(head) is not str:
482
raise TypeError(head)
484
o = lookup_object(head)
123
487
if isinstance(o, Commit):
124
rev = mapping.import_commit(o)
125
root_trees[rev] = objects[o.tree_sha]
488
rev, roundtrip_revid, verifiers = mapping.import_commit(o,
489
mapping.revision_id_foreign_to_bzr)
490
if (repo.has_revision(rev.revision_id) or
491
(roundtrip_revid and repo.has_revision(roundtrip_revid))):
493
graph.append((o.id, o.parents))
494
heads.extend([p for p in o.parents if p not in checked])
495
elif isinstance(o, Tag):
496
if o.object[1] not in checked:
497
heads.append(o.object[1])
499
trace.warning("Unable to import head object %r" % o)
502
# Order the revisions
126
503
# Create the inventory objects
127
for rev, root_tree in root_trees.iteritems():
128
# We have to do this here, since we have to walk the tree and
129
# we need to make sure to import the blobs / trees with the riht
130
# path; this may involve adding them more than once.
132
def lookup_object(sha):
135
return reconstruct_git_object(repo, mapping, sha)
136
import_git_tree(repo, mapping, "", tree, inv, lookup_object)
137
repo.add_revision(rev.revision_id, rev, inv)
140
def reconstruct_git_commit(repo, rev):
141
raise NotImplementedError(self.reconstruct_git_commit)
144
def reconstruct_git_object(repo, mapping, sha):
146
revid = mapping.revision_id_foreign_to_bzr(sha)
148
rev = repo.get_revision(revid)
149
except NoSuchRevision:
152
return reconstruct_git_commit(rev)
156
raise KeyError("No such object %s" % sha)
159
class InterGitRepository(InterRepository):
161
_matching_repo_format = GitFormat()
164
def _get_repo_format_to_test():
167
def copy_content(self, revision_id=None, pb=None):
168
"""See InterRepository.copy_content."""
169
self.fetch(revision_id, pb, find_ghosts=False)
171
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
174
mapping = self.source.get_mapping()
177
pb.note("git: %s" % text)
179
info("git: %s" % text)
180
def determine_wants(heads):
181
if revision_id is None:
184
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
185
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
186
graph_walker = BzrFetchGraphWalker(self.target, mapping)
187
self.target.lock_write()
505
revision_ids = topo_sort(graph)
507
if limit is not None:
508
revision_ids = revision_ids[:limit]
510
for offset in range(0, len(revision_ids), batch_size):
511
target_git_object_retriever.start_write_group()
189
import_git_objects(self.target, mapping,
190
self.source.fetch_objects(determine_wants, graph_walker,
196
def is_compatible(source, target):
197
"""Be compatible with GitRepository."""
198
# FIXME: Also check target uses VersionedFile
199
return (isinstance(source, LocalGitRepository) and
200
target.supports_rich_root())
513
repo.start_write_group()
515
for i, head in enumerate(
516
revision_ids[offset:offset+batch_size]):
518
pb.update("fetching revisions", offset+i,
520
import_git_commit(repo, mapping, head, lookup_object,
521
target_git_object_retriever, trees_cache)
524
repo.abort_write_group()
527
hint = repo.commit_write_group()
529
pack_hints.extend(hint)
531
target_git_object_retriever.abort_write_group()
534
target_git_object_retriever.commit_write_group()
535
return pack_hints, last_imported
538
class DetermineWantsRecorder(object):
540
def __init__(self, actual):
543
self.remote_refs = {}
545
def __call__(self, refs):
546
if type(refs) is not dict:
547
raise TypeError(refs)
548
self.remote_refs = refs
549
self.wants = self.actual(refs)
added
removed
breezy/plugins/git/fetch.py
