14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
24
from bzrlib.plugins.git import git
17
from cStringIO import (
21
from dulwich.client import (
22
SimpleFetchGraphWalker,
24
from dulwich.objects import (
35
from bzrlib.errors import (
39
from bzrlib.inventory import (
45
from bzrlib.lru_cache import (
48
from bzrlib.repository import (
51
from bzrlib.revision import (
54
from bzrlib.tsort import (
58
from bzrlib.plugins.git.converter import (
25
61
from bzrlib.plugins.git.repository import (
30
from bzrlib.plugins.git.remote import RemoteGitRepository
32
from dulwich.objects import Commit
34
from cStringIO import StringIO
66
from bzrlib.plugins.git.remote import (
37
71
class BzrFetchGraphWalker(object):
72
"""GraphWalker implementation that uses a Bazaar repository."""
39
74
def __init__(self, repository, mapping):
40
75
self.repository = repository
63
101
self.heads.update([p for p in ps if not p in self.done])
65
103
self.done.add(ret)
66
return self.mapping.revision_id_bzr_to_foreign(ret)
104
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
67
105
except InvalidRevisionId:
72
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, executable):
110
def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,
111
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
73
112
"""Import a git blob object into a bzr repository.
75
:param repo: bzr repository
114
:param texts: VersionedFiles to add to
76
115
:param path: Path in the tree
77
116
:param blob: A git blob
117
:return: Inventory delta for this file
79
119
file_id = mapping.generate_file_id(path)
80
text_revision = inv.revision_id
81
repo.texts.add_lines((file_id, text_revision),
82
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
83
osutils.split_lines(blob.data))
84
ie = inv.add_path(path, "file", file_id)
85
ie.revision = text_revision
86
ie.text_size = len(blob.data)
87
ie.text_sha1 = osutils.sha_string(blob.data)
124
# We just have to hope this is indeed utf-8:
125
ie = cls(file_id, urlutils.basename(path).decode("utf-8"),
88
127
ie.executable = executable
91
def import_git_tree(repo, mapping, path, tree, inv, parent_invs, lookup_object):
128
# See if this has changed at all
130
base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)
134
if (base_sha == hexsha and base_inv[file_id].executable == ie.executable
135
and base_inv[file_id].kind == ie.kind):
136
# If nothing has changed since the base revision, we're done
138
if base_sha == hexsha:
139
ie.text_size = base_inv[file_id].text_size
140
ie.text_sha1 = base_inv[file_id].text_sha1
141
ie.symlink_target = base_inv[file_id].symlink_target
142
ie.revision = base_inv[file_id].revision
144
blob = lookup_object(hexsha)
145
if ie.kind == "symlink":
146
ie.symlink_target = blob.data
150
ie.text_size = len(blob.data)
151
ie.text_sha1 = osutils.sha_string(blob.data)
152
# Check what revision we should store
154
for pinv in parent_invs:
155
if not file_id in pinv:
157
if pinv[file_id].text_sha1 == ie.text_sha1:
158
# found a revision in one of the parents to use
159
ie.revision = pinv[file_id].revision
161
parent_keys.append((file_id, pinv[file_id].revision))
162
if ie.revision is None:
163
# Need to store a new revision
164
ie.revision = revision_id
165
assert file_id is not None
166
assert ie.revision is not None
167
texts.add_lines((file_id, ie.revision), parent_keys,
168
osutils.split_lines(blob.data))
169
shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))
170
if file_id in base_inv:
171
old_path = base_inv.id2path(file_id)
174
return [(old_path, path, file_id, ie)]
177
def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,
178
revision_id, parent_invs, shagitmap, lookup_object):
92
179
"""Import a git tree object into a bzr repository.
94
:param repo: A Bzr repository object
181
:param texts: VersionedFiles object to add to
95
182
:param path: Path in the tree
96
183
:param tree: A git tree object
97
:param inv: Inventory object
184
:param base_inv: Base inventory against which to return inventory delta
185
:return: Inventory delta for this subtree
99
188
file_id = mapping.generate_file_id(path)
100
text_revision = inv.revision_id
101
repo.texts.add_lines((file_id, text_revision),
102
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
104
ie = inv.add_path(path, "directory", file_id)
105
ie.revision = text_revision
189
# We just have to hope this is indeed utf-8:
190
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
192
if not file_id in base_inv:
193
# Newly appeared here
194
ie.revision = revision_id
195
texts.add_lines((file_id, ie.revision), [], [])
196
ret.append((None, path, file_id, ie))
198
# See if this has changed at all
200
base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)
204
if base_sha == hexsha:
205
# If nothing has changed since the base revision, we're done
207
# Remember for next time
208
existing_children = set()
209
shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))
210
tree = lookup_object(hexsha)
106
211
for mode, name, hexsha in tree.entries():
107
212
entry_kind = (mode & 0700000) / 0100000
108
213
basename = name.decode("utf-8")
214
existing_children.add(basename)
110
216
child_path = name
112
218
child_path = urlutils.join(path, name)
113
219
if entry_kind == 0:
114
tree = lookup_object(hexsha)
115
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, lookup_object)
220
ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,
221
file_id, revision_id, parent_invs, shagitmap, lookup_object))
116
222
elif entry_kind == 1:
117
blob = lookup_object(hexsha)
118
223
fs_mode = mode & 0777
119
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, bool(fs_mode & 0111))
224
file_kind = (mode & 070000) / 010000
225
if file_kind == 0: # regular file
230
raise AssertionError("Unknown file kind, mode=%r" % (mode,))
231
ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,
232
file_id, revision_id, parent_invs, shagitmap, lookup_object,
233
bool(fs_mode & 0111), symlink))
121
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
124
def import_git_objects(repo, mapping, object_iter, pb=None):
235
raise AssertionError("Unknown object kind, perms=%r." % (mode,))
236
# Remove any children that have disappeared
237
if file_id in base_inv:
238
deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]
241
ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))
242
if ie.kind == "directory":
243
deletable.extend(ie.children.values())
247
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
125
249
"""Import a set of git objects into a bzr repository.
127
251
:param repo: Bazaar repository
129
253
:param object_iter: Iterator over Git objects.
131
255
# TODO: a more (memory-)efficient implementation of this
133
for i, o in enumerate(object_iter):
135
pb.update("fetching objects", i)
261
parent_invs_cache = LRUCache(50)
140
262
# Find and convert commit objects
141
for o in objects.itervalues():
265
pb.update("finding revisions to fetch", len(graph), None)
267
assert isinstance(head, str)
269
o = object_iter[head]
142
272
if isinstance(o, Commit):
143
273
rev = mapping.import_commit(o)
144
root_trees[rev.revision_id] = objects[o.tree]
274
if repo.has_revision(rev.revision_id):
276
root_trees[rev.revision_id] = o.tree
145
277
revisions[rev.revision_id] = rev
146
278
graph.append((rev.revision_id, rev.parent_ids))
279
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),
280
"commit", (rev.revision_id, o._tree))
281
heads.extend([p for p in o.parents if p not in checked])
282
elif isinstance(o, Tag):
283
heads.append(o.object[1])
285
trace.warning("Unable to import head object %r" % o)
147
287
# Order the revisions
148
288
# Create the inventory objects
149
289
for i, revid in enumerate(topo_sort(graph)):
150
290
if pb is not None:
151
291
pb.update("fetching revisions", i, len(graph))
152
root_tree = root_trees[revid]
153
292
rev = revisions[revid]
154
293
# We have to do this here, since we have to walk the tree and
155
# we need to make sure to import the blobs / trees with the riht
294
# we need to make sure to import the blobs / trees with the right
156
295
# path; this may involve adding them more than once.
158
inv.revision_id = rev.revision_id
159
296
def lookup_object(sha):
162
return reconstruct_git_object(repo, mapping, sha)
163
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
164
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs, lookup_object)
165
repo.add_revision(rev.revision_id, rev, inv)
168
def reconstruct_git_commit(repo, rev):
169
raise NotImplementedError(self.reconstruct_git_commit)
172
def reconstruct_git_object(repo, mapping, sha):
174
revid = mapping.revision_id_foreign_to_bzr(sha)
176
rev = repo.get_revision(revid)
177
except NoSuchRevision:
180
return reconstruct_git_commit(rev)
184
raise KeyError("No such object %s" % sha)
298
return object_iter[sha]
300
return target_git_object_retriever[sha]
302
for parent_id in rev.parent_ids:
304
parent_invs.append(parent_invs_cache[parent_id])
306
parent_inv = repo.get_inventory(parent_id)
307
parent_invs.append(parent_inv)
308
parent_invs_cache[parent_id] = parent_inv
309
if parent_invs == []:
310
base_inv = Inventory(root_id=None)
312
base_inv = parent_invs[0]
313
inv_delta = import_git_tree(repo.texts, mapping, "",
314
root_trees[revid], base_inv, None, revid, parent_invs,
315
target_git_object_retriever._idmap, lookup_object)
317
basis_id = rev.parent_ids[0]
319
basis_id = NULL_REVISION
320
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
321
inv_delta, rev.revision_id, rev.parent_ids)
322
parent_invs_cache[rev.revision_id] = inv
323
repo.add_revision(rev.revision_id, rev)
324
target_git_object_retriever._idmap.commit()
327
class InterGitNonGitRepository(InterRepository):
328
"""Base InterRepository that copies revisions from a Git into a non-Git
331
_matching_repo_format = GitRepositoryFormat()
334
def _get_repo_format_to_test():
337
def copy_content(self, revision_id=None, pb=None):
338
"""See InterRepository.copy_content."""
339
self.fetch(revision_id, pb, find_ghosts=False)
341
def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,
343
self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,
344
mapping=mapping, fetch_spec=fetch_spec)
346
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
347
mapping=None, fetch_spec=None):
349
mapping = self.source.get_mapping()
350
if revision_id is not None:
351
interesting_heads = [revision_id]
352
elif fetch_spec is not None:
353
interesting_heads = fetch_spec.heads
355
interesting_heads = None
357
def determine_wants(refs):
359
if interesting_heads is None:
360
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
362
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]
363
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
364
self.fetch_objects(determine_wants, mapping, pb)
369
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
370
"""InterRepository that copies revisions from a remote Git into a non-Git
373
def fetch_objects(self, determine_wants, mapping, pb=None):
375
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
376
graph_walker = BzrFetchGraphWalker(self.target, mapping)
379
create_pb = pb = ui.ui_factory.nested_progress_bar()
380
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
383
def record_determine_wants(heads):
384
wants = determine_wants(heads)
385
recorded_wants.extend(wants)
389
self.target.lock_write()
391
self.target.start_write_group()
393
objects_iter = self.source.fetch_objects(
394
record_determine_wants,
396
target_git_object_retriever.get_raw,
398
import_git_objects(self.target, mapping, objects_iter,
399
target_git_object_retriever, recorded_wants, pb)
401
self.target.commit_write_group()
409
def is_compatible(source, target):
410
"""Be compatible with GitRepository."""
411
# FIXME: Also check target uses VersionedFile
412
return (isinstance(source, RemoteGitRepository) and
413
target.supports_rich_root() and
414
not isinstance(target, GitRepository))
417
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
418
"""InterRepository that copies revisions from a remote Git into a non-Git
421
def fetch_objects(self, determine_wants, mapping, pb=None):
422
wants = determine_wants(self.source._git.get_refs())
425
create_pb = pb = ui.ui_factory.nested_progress_bar()
426
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
428
self.target.lock_write()
430
self.target.start_write_group()
432
import_git_objects(self.target, mapping,
433
self.source._git.object_store,
434
target_git_object_retriever, wants, pb)
436
self.target.commit_write_group()
444
def is_compatible(source, target):
445
"""Be compatible with GitRepository."""
446
# FIXME: Also check target uses VersionedFile
447
return (isinstance(source, LocalGitRepository) and
448
target.supports_rich_root() and
449
not isinstance(target, GitRepository))
187
452
class InterGitRepository(InterRepository):
453
"""InterRepository that copies between Git repositories."""
189
_matching_repo_format = GitFormat()
455
_matching_repo_format = GitRepositoryFormat()
192
458
def _get_repo_format_to_test():
197
463
self.fetch(revision_id, pb, find_ghosts=False)
199
465
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
466
mapping=None, fetch_spec=None):
201
467
if mapping is None:
202
468
mapping = self.source.get_mapping()
203
469
def progress(text):
204
pb.note("git: %s", text)
205
def determine_wants(heads):
206
if revision_id is None:
209
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
210
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
211
graph_walker = BzrFetchGraphWalker(self.target, mapping)
214
create_pb = pb = ui.ui_factory.nested_progress_bar()
470
trace.info("git: %s", text)
472
if revision_id is not None:
473
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
474
elif fetch_spec is not None:
475
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
476
if fetch_spec is None and revision_id is None:
477
determine_wants = r.object_store.determine_wants_all
479
determine_wants = lambda x: [y for y in args if not y in r.object_store]
481
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
482
f, commit = r.object_store.add_pack()
216
self.target.lock_write()
218
self.target.start_write_group()
220
import_git_objects(self.target, mapping,
221
iter(self.source.fetch_objects(determine_wants, graph_walker,
224
self.target.commit_write_group()
484
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
232
492
def is_compatible(source, target):
233
493
"""Be compatible with GitRepository."""
234
# FIXME: Also check target uses VersionedFile
235
494
return (isinstance(source, GitRepository) and
236
target.supports_rich_root())
495
isinstance(target, GitRepository))