14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from bzrlib import osutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
23
from bzrlib.plugins.git import git
24
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
25
from bzrlib.plugins.git.remote import RemoteGitRepository
27
from dulwich.objects import Commit
29
from cStringIO import StringIO
32
class BzrFetchGraphWalker(object):
34
def __init__(self, repository, mapping):
35
self.repository = repository
36
self.mapping = mapping
38
self.heads = set(repository.all_revision_ids())
42
revid = self.mapping.revision_id_foreign_to_bzr(sha)
45
def remove(self, revid):
48
self.heads.remove(revid)
49
if revid in self.parents:
50
for p in self.parents[revid]:
55
ret = self.heads.pop()
56
ps = self.repository.get_parent_map([ret])[ret]
57
self.parents[ret] = ps
58
self.heads.update([p for p in ps if not p in self.done])
61
return self.mapping.revision_id_bzr_to_foreign(ret)
62
except InvalidRevisionId:
67
def import_git_blob(repo, mapping, path, blob):
17
from dulwich.objects import (
23
from dulwich.object_store import (
26
from itertools import (
39
from bzrlib.errors import (
43
from bzrlib.inventory import (
50
from bzrlib.repository import (
53
from bzrlib.revision import (
56
from bzrlib.revisiontree import (
59
from bzrlib.tsort import (
62
from bzrlib.versionedfile import (
63
ChunkedContentFactory,
66
from bzrlib.plugins.git.mapping import (
72
from bzrlib.plugins.git.object_store import (
77
from bzrlib.plugins.git.remote import (
80
from bzrlib.plugins.git.repository import (
87
def import_git_blob(texts, mapping, path, name, (base_hexsha, hexsha),
88
base_inv, parent_id, revision_id,
89
parent_invs, lookup_object, (base_mode, mode), store_updater,
68
91
"""Import a git blob object into a bzr repository.
70
:param repo: bzr repository
93
:param texts: VersionedFiles to add to
71
94
:param path: Path in the tree
72
95
:param blob: A git blob
96
:return: Inventory delta for this file
74
file_id = mapping.generate_file_id(path)
75
repo.texts.add_lines((file_id, blob.id),
77
osutils.split_lines(blob.data))
78
inv.add_path(path, "file", file_id)
81
def import_git_tree(repo, mapping, path, tree, inv, lookup_object):
98
if mapping.is_control_file(path):
100
if base_hexsha == hexsha and base_mode == mode:
101
# If nothing has changed since the base revision, we're done
103
file_id = lookup_file_id(path)
104
if stat.S_ISLNK(mode):
108
ie = cls(file_id, name.decode("utf-8"), parent_id)
109
ie.executable = mode_is_executable(mode)
110
if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):
111
base_ie = base_inv[base_inv.path2id(path)]
112
ie.text_size = base_ie.text_size
113
ie.text_sha1 = base_ie.text_sha1
114
ie.symlink_target = base_ie.symlink_target
115
if ie.executable == base_ie.executable:
116
ie.revision = base_ie.revision
118
blob = lookup_object(hexsha)
120
blob = lookup_object(hexsha)
121
if ie.kind == "symlink":
123
ie.symlink_target = blob.data
127
ie.text_size = sum(imap(len, blob.chunked))
128
ie.text_sha1 = osutils.sha_strings(blob.chunked)
129
# Check what revision we should store
131
for pinv in parent_invs:
136
if (pie.text_sha1 == ie.text_sha1 and
137
pie.executable == ie.executable and
138
pie.symlink_target == ie.symlink_target):
139
# found a revision in one of the parents to use
140
ie.revision = pie.revision
142
parent_key = (file_id, pie.revision)
143
if not parent_key in parent_keys:
144
parent_keys.append(parent_key)
145
if ie.revision is None:
146
# Need to store a new revision
147
ie.revision = revision_id
148
assert ie.revision is not None
149
if ie.kind == 'symlink':
152
chunks = blob.chunked
153
texts.insert_record_stream([
154
ChunkedContentFactory((file_id, ie.revision),
155
tuple(parent_keys), ie.text_sha1, chunks)])
157
if base_hexsha is not None:
158
old_path = path # Renames are not supported yet
159
if stat.S_ISDIR(base_mode):
160
invdelta.extend(remove_disappeared_children(base_inv, old_path,
161
lookup_object(base_hexsha), [], lookup_object))
164
invdelta.append((old_path, path, file_id, ie))
165
if base_hexsha != hexsha:
166
store_updater.add_object(blob, ie)
170
class SubmodulesRequireSubtrees(BzrError):
171
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
175
def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),
176
base_inv, parent_id, revision_id, parent_invs, lookup_object,
177
(base_mode, mode), store_updater, lookup_file_id):
178
if base_hexsha == hexsha and base_mode == mode:
180
file_id = lookup_file_id(path)
181
ie = TreeReference(file_id, name.decode("utf-8"), parent_id)
182
ie.revision = revision_id
183
if base_hexsha is None:
187
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
188
texts.insert_record_stream([
189
ChunkedContentFactory((file_id, ie.revision), (), None, [])])
190
invdelta = [(oldpath, path, file_id, ie)]
194
def remove_disappeared_children(base_inv, path, base_tree, existing_children,
197
for name, mode, hexsha in base_tree.iteritems():
198
if name in existing_children:
200
c_path = posixpath.join(path, name.decode("utf-8"))
201
ret.append((c_path, None, base_inv.path2id(c_path), None))
202
if stat.S_ISDIR(mode):
203
ret.extend(remove_disappeared_children(
204
base_inv, c_path, lookup_object(hexsha), [], lookup_object))
208
def import_git_tree(texts, mapping, path, name, (base_hexsha, hexsha),
209
base_inv, parent_id, revision_id, parent_invs,
210
lookup_object, (base_mode, mode), store_updater,
211
lookup_file_id, allow_submodules=False):
82
212
"""Import a git tree object into a bzr repository.
84
:param repo: A Bzr repository object
214
:param texts: VersionedFiles object to add to
85
215
:param path: Path in the tree
86
216
:param tree: A git tree object
87
:param inv: Inventory object
217
:param base_inv: Base inventory against which to return inventory delta
218
:return: Inventory delta for this subtree
89
file_id = mapping.generate_file_id(path)
90
repo.texts.add_lines((file_id, tree.id),
93
inv.add_path(path, "directory", file_id)
94
for mode, name, hexsha in tree.entries():
95
entry_kind = (mode & 0700000) / 0100000
96
basename = name.decode("utf-8")
100
child_path = urlutils.join(path, name)
102
import_git_tree(repo, mapping, child_path, lookup_object, inv)
103
elif entry_kind == 1:
104
import_git_blob(repo, mapping, child_path, lookup_object, inv)
106
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
109
def import_git_objects(repo, mapping, object_iter):
220
if base_hexsha == hexsha and base_mode == mode:
221
# If nothing has changed since the base revision, we're done
224
file_id = lookup_file_id(path)
225
# We just have to hope this is indeed utf-8:
226
ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)
227
tree = lookup_object(hexsha)
228
if base_hexsha is None:
230
old_path = None # Newly appeared here
232
base_tree = lookup_object(base_hexsha)
233
old_path = path # Renames aren't supported yet
234
if base_tree is None or type(base_tree) is not Tree:
235
ie.revision = revision_id
236
invdelta.append((old_path, path, ie.file_id, ie))
237
texts.insert_record_stream([
238
ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
239
# Remember for next time
240
existing_children = set()
242
for child_mode, name, child_hexsha in tree.entries():
243
existing_children.add(name)
244
child_path = posixpath.join(path, name)
245
if type(base_tree) is Tree:
247
child_base_mode, child_base_hexsha = base_tree[name]
249
child_base_hexsha = None
252
child_base_hexsha = None
254
if stat.S_ISDIR(child_mode):
255
subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
256
child_path, name, (child_base_hexsha, child_hexsha), base_inv,
257
file_id, revision_id, parent_invs, lookup_object,
258
(child_base_mode, child_mode), store_updater, lookup_file_id,
259
allow_submodules=allow_submodules)
260
elif S_ISGITLINK(child_mode): # submodule
261
if not allow_submodules:
262
raise SubmodulesRequireSubtrees()
263
subinvdelta, grandchildmodes = import_git_submodule(texts, mapping,
264
child_path, name, (child_base_hexsha, child_hexsha), base_inv,
265
file_id, revision_id, parent_invs, lookup_object,
266
(child_base_mode, child_mode), store_updater, lookup_file_id)
268
subinvdelta = import_git_blob(texts, mapping, child_path, name,
269
(child_base_hexsha, child_hexsha), base_inv, file_id,
270
revision_id, parent_invs, lookup_object,
271
(child_base_mode, child_mode), store_updater, lookup_file_id)
273
child_modes.update(grandchildmodes)
274
invdelta.extend(subinvdelta)
275
if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
276
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
277
child_modes[child_path] = child_mode
278
# Remove any children that have disappeared
279
if base_tree is not None and type(base_tree) is Tree:
280
invdelta.extend(remove_disappeared_children(base_inv, old_path,
281
base_tree, existing_children, lookup_object))
282
store_updater.add_object(tree, ie)
283
return invdelta, child_modes
286
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
287
o, rev, ret_tree, parent_trees, mapping, unusual_modes):
288
new_unusual_modes = mapping.export_unusual_file_modes(rev)
289
if new_unusual_modes != unusual_modes:
290
raise AssertionError("unusual modes don't match: %r != %r" % (
291
unusual_modes, new_unusual_modes))
292
# Verify that we can reconstruct the commit properly
293
rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree)
295
raise AssertionError("Reconstructed commit differs: %r != %r" % (
299
for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,
300
target_git_object_retriever._cache.idmap, unusual_modes, mapping.BZR_DUMMY_FILE):
301
old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]
303
if obj.id != old_obj_id:
304
diff.append((path, lookup_object(old_obj_id), obj))
305
for (path, old_obj, new_obj) in diff:
306
while (old_obj.type_name == "tree" and
307
new_obj.type_name == "tree" and
308
sorted(old_obj) == sorted(new_obj)):
310
if old_obj[name][0] != new_obj[name][0]:
311
raise AssertionError("Modes for %s differ: %o != %o" %
312
(path, old_obj[name][0], new_obj[name][0]))
313
if old_obj[name][1] != new_obj[name][1]:
314
# Found a differing child, delve deeper
315
path = posixpath.join(path, name)
316
old_obj = lookup_object(old_obj[name][1])
317
new_obj = new_objs[path]
319
raise AssertionError("objects differ for %s: %r != %r" % (path,
323
def import_git_commit(repo, mapping, head, lookup_object,
324
target_git_object_retriever, trees_cache):
325
o = lookup_object(head)
326
rev = mapping.import_commit(o,
327
lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])
328
# We have to do this here, since we have to walk the tree and
329
# we need to make sure to import the blobs / trees with the right
330
# path; this may involve adding them more than once.
331
parent_trees = trees_cache.revision_trees(rev.parent_ids)
332
if parent_trees == []:
333
base_inv = Inventory(root_id=None)
337
base_inv = parent_trees[0].inventory
338
base_tree = lookup_object(o.parents[0]).tree
339
base_mode = stat.S_IFDIR
340
store_updater = target_git_object_retriever._get_updater(rev)
341
store_updater.add_object(o, None)
342
lookup_file_id = mapping.get_fileid_map(lookup_object, o.tree).lookup_file_id
343
inv_delta, unusual_modes = import_git_tree(repo.texts,
344
mapping, "", u"", (base_tree, o.tree), base_inv,
345
None, rev.revision_id, [p.inventory for p in parent_trees],
346
lookup_object, (base_mode, stat.S_IFDIR), store_updater,
348
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
349
store_updater.finish()
350
if unusual_modes != {}:
351
for path, mode in unusual_modes.iteritems():
352
warn_unusual_mode(rev.foreign_revid, path, mode)
353
mapping.import_unusual_file_modes(rev, unusual_modes)
355
basis_id = rev.parent_ids[0]
357
basis_id = NULL_REVISION
359
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
360
inv_delta, rev.revision_id, rev.parent_ids,
362
ret_tree = RevisionTree(repo, inv, rev.revision_id)
363
trees_cache.add(ret_tree)
364
repo.add_revision(rev.revision_id, rev)
365
if "verify" in debug.debug_flags:
366
verify_commit_reconstruction(target_git_object_retriever,
367
lookup_object, o, rev, ret_tree, parent_trees, mapping,
371
def import_git_objects(repo, mapping, object_iter,
372
target_git_object_retriever, heads, pb=None, limit=None):
110
373
"""Import a set of git objects into a bzr repository.
112
:param repo: Bazaar repository
375
:param repo: Target Bazaar repository
113
376
:param mapping: Mapping to use
114
377
:param object_iter: Iterator over Git objects.
378
:return: Tuple with pack hints and last imported revision id
116
# TODO: a more (memory-)efficient implementation of this
118
for o in object_iter:
380
def lookup_object(sha):
382
return object_iter[sha]
384
return target_git_object_retriever[sha]
387
heads = list(set(heads))
388
trees_cache = LRUTreeCache(repo)
121
389
# Find and convert commit objects
122
for o in objects.iterkeys():
392
pb.update("finding revisions to fetch", len(graph), None)
394
assert isinstance(head, str)
396
o = lookup_object(head)
123
399
if isinstance(o, Commit):
124
rev = mapping.import_commit(o)
125
root_trees[rev] = objects[o.tree_sha]
400
rev = mapping.import_commit(o, lambda x: None)
401
if repo.has_revision(rev.revision_id):
403
graph.append((o.id, o.parents))
404
heads.extend([p for p in o.parents if p not in checked])
405
elif isinstance(o, Tag):
406
if o.object[1] not in checked:
407
heads.append(o.object[1])
409
trace.warning("Unable to import head object %r" % o)
412
# Order the revisions
126
413
# Create the inventory objects
127
for rev, root_tree in root_trees.iteritems():
128
# We have to do this here, since we have to walk the tree and
129
# we need to make sure to import the blobs / trees with the riht
130
# path; this may involve adding them more than once.
132
def lookup_object(sha):
135
return reconstruct_git_object(repo, mapping, sha)
136
import_git_tree(repo, mapping, "", tree, inv, lookup_object)
137
repo.add_revision(rev.revision_id, rev, inv)
140
def reconstruct_git_commit(repo, rev):
141
raise NotImplementedError(self.reconstruct_git_commit)
144
def reconstruct_git_object(repo, mapping, sha):
146
revid = mapping.revision_id_foreign_to_bzr(sha)
148
rev = repo.get_revision(revid)
149
except NoSuchRevision:
152
return reconstruct_git_commit(rev)
156
raise KeyError("No such object %s" % sha)
415
revision_ids = topo_sort(graph)
417
if limit is not None:
418
revision_ids = revision_ids[:limit]
420
for offset in range(0, len(revision_ids), batch_size):
421
target_git_object_retriever.start_write_group()
423
repo.start_write_group()
425
for i, head in enumerate(
426
revision_ids[offset:offset+batch_size]):
428
pb.update("fetching revisions", offset+i,
430
import_git_commit(repo, mapping, head, lookup_object,
431
target_git_object_retriever, trees_cache)
434
repo.abort_write_group()
437
hint = repo.commit_write_group()
439
pack_hints.extend(hint)
441
target_git_object_retriever.abort_write_group()
444
target_git_object_retriever.commit_write_group()
445
return pack_hints, last_imported
159
448
class InterGitRepository(InterRepository):
161
_matching_repo_format = GitFormat()
450
_matching_repo_format = GitRepositoryFormat()
164
453
def _get_repo_format_to_test():
168
457
"""See InterRepository.copy_content."""
169
458
self.fetch(revision_id, pb, find_ghosts=False)
171
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
460
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
461
mapping=None, fetch_spec=None):
462
self.fetch_refs(revision_id=revision_id, pb=pb,
463
find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)
466
class InterGitNonGitRepository(InterGitRepository):
467
"""Base InterRepository that copies revisions from a Git into a non-Git
470
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
471
mapping=None, fetch_spec=None):
173
472
if mapping is None:
174
473
mapping = self.source.get_mapping()
474
if revision_id is not None:
475
interesting_heads = [revision_id]
476
elif fetch_spec is not None:
477
interesting_heads = fetch_spec.heads
479
interesting_heads = None
481
def determine_wants(refs):
483
if interesting_heads is None:
484
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
486
ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
487
return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]
488
(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)
489
if pack_hint is not None and self.target._format.pack_compresses:
490
self.target.pack(hint=pack_hint)
494
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
495
def report_git_progress(pb, text):
496
text = text.rstrip("\r\n")
497
g = _GIT_PROGRESS_RE.match(text)
499
(text, pct, current, total) = g.groups()
500
pb.update(text, int(current), int(total))
502
pb.update(text, 0, 0)
505
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
506
"""InterRepository that copies revisions from a remote Git into a non-Git
509
def get_target_heads(self):
510
# FIXME: This should be more efficient
511
all_revs = self.target.all_revision_ids()
512
parent_map = self.target.get_parent_map(all_revs)
514
map(all_parents.update, parent_map.itervalues())
515
return set(all_revs) - all_parents
517
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
175
518
def progress(text):
177
pb.note("git: %s" % text)
179
info("git: %s" % text)
180
def determine_wants(heads):
181
if revision_id is None:
184
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
185
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
186
graph_walker = BzrFetchGraphWalker(self.target, mapping)
519
report_git_progress(pb, text)
520
store = BazaarObjectStore(self.target, mapping)
187
521
self.target.lock_write()
189
import_git_objects(self.target, mapping,
190
self.source.fetch_objects(determine_wants, graph_walker,
523
heads = self.get_target_heads()
524
graph_walker = store.get_graph_walker(
525
[store._lookup_revision_sha1(head) for head in heads])
528
def record_determine_wants(heads):
529
wants = determine_wants(heads)
530
recorded_wants.extend(wants)
535
create_pb = pb = ui.ui_factory.nested_progress_bar()
537
objects_iter = self.source.fetch_objects(record_determine_wants,
538
graph_walker, store.get_raw, progress)
539
return import_git_objects(self.target, mapping,
540
objects_iter, store, recorded_wants, pb, limit)
193
545
self.target.unlock()
196
548
def is_compatible(source, target):
197
549
"""Be compatible with GitRepository."""
198
# FIXME: Also check target uses VersionedFile
199
return (isinstance(source, LocalGitRepository) and
200
target.supports_rich_root())
550
return (isinstance(source, RemoteGitRepository) and
551
target.supports_rich_root() and
552
not isinstance(target, GitRepository) and
553
target.texts is not None)
556
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
557
"""InterRepository that copies revisions from a local Git into a non-Git
560
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
563
wants = determine_wants(self.source._git.get_refs())
566
create_pb = pb = ui.ui_factory.nested_progress_bar()
567
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
569
self.target.lock_write()
571
return import_git_objects(self.target, mapping,
572
self.source._git.object_store,
573
target_git_object_retriever, wants, pb, limit)
581
def is_compatible(source, target):
582
"""Be compatible with GitRepository."""
583
return (isinstance(source, LocalGitRepository) and
584
target.supports_rich_root() and
585
not isinstance(target, GitRepository) and
586
target.texts is not None)
589
class InterGitGitRepository(InterGitRepository):
590
"""InterRepository that copies between Git repositories."""
592
def fetch_objects(self, determine_wants, mapping, pb=None):
594
trace.note("git: %s", text)
595
graphwalker = self.target._git.get_graph_walker()
596
if (isinstance(self.source, LocalGitRepository) and
597
isinstance(self.target, LocalGitRepository)):
598
return self.source._git.fetch(self.target._git, determine_wants,
600
elif (isinstance(self.source, LocalGitRepository) and
601
isinstance(self.target, RemoteGitRepository)):
602
raise NotImplementedError
603
elif (isinstance(self.source, RemoteGitRepository) and
604
isinstance(self.target, LocalGitRepository)):
605
f, commit = self.target._git.object_store.add_thin_pack()
607
refs = self.source._git.fetch_pack(determine_wants,
608
graphwalker, f.write, progress)
617
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
618
mapping=None, fetch_spec=None, branches=None):
620
mapping = self.source.get_mapping()
622
if revision_id is not None:
623
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
624
elif fetch_spec is not None:
625
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
626
if branches is not None:
627
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
628
elif fetch_spec is None and revision_id is None:
629
determine_wants = r.object_store.determine_wants_all
631
determine_wants = lambda x: [y for y in args if not y in r.object_store]
632
return self.fetch_objects(determine_wants, mapping)[0]
636
def is_compatible(source, target):
637
"""Be compatible with GitRepository."""
638
return (isinstance(source, GitRepository) and
639
isinstance(target, GitRepository))
added
removed
fetch.py
