14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from bzrlib import osutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
23
from bzrlib.plugins.git import git
24
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
25
from bzrlib.plugins.git.remote import RemoteGitRepository
27
from dulwich.objects import Commit
29
from cStringIO import StringIO
17
from cStringIO import (
21
from dulwich.client import (
22
SimpleFetchGraphWalker,
24
from dulwich.objects import (
36
from bzrlib.errors import (
40
from bzrlib.inventory import (
46
from bzrlib.lru_cache import (
49
from bzrlib.repository import (
52
from bzrlib.revision import (
55
from bzrlib.tsort import (
59
from bzrlib.plugins.git.converter import (
62
from bzrlib.plugins.git.mapping import (
68
from bzrlib.plugins.git.repository import (
73
from bzrlib.plugins.git.remote import (
32
78
class BzrFetchGraphWalker(object):
79
"""GraphWalker implementation that uses a Bazaar repository."""
34
81
def __init__(self, repository, mapping):
35
82
self.repository = repository
58
108
self.heads.update([p for p in ps if not p in self.done])
60
110
self.done.add(ret)
61
return self.mapping.revision_id_bzr_to_foreign(ret)
111
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
62
112
except InvalidRevisionId:
67
def import_git_blob(repo, mapping, path, blob):
117
def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,
118
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
68
119
"""Import a git blob object into a bzr repository.
70
:param repo: bzr repository
121
:param texts: VersionedFiles to add to
71
122
:param path: Path in the tree
72
123
:param blob: A git blob
124
:return: Inventory delta for this file
74
126
file_id = mapping.generate_file_id(path)
75
repo.texts.add_lines((file_id, blob.id),
77
osutils.split_lines(blob.data))
78
inv.add_path(path, "file", file_id)
81
def import_git_tree(repo, mapping, path, tree, inv, lookup_object):
131
# We just have to hope this is indeed utf-8:
132
ie = cls(file_id, urlutils.basename(path).decode("utf-8"),
134
ie.executable = executable
135
# See if this has changed at all
137
base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)
141
if (base_sha == hexsha and base_inv[file_id].executable == ie.executable
142
and base_inv[file_id].kind == ie.kind):
143
# If nothing has changed since the base revision, we're done
145
if base_sha == hexsha:
146
ie.text_size = base_inv[file_id].text_size
147
ie.text_sha1 = base_inv[file_id].text_sha1
148
ie.symlink_target = base_inv[file_id].symlink_target
149
ie.revision = base_inv[file_id].revision
151
blob = lookup_object(hexsha)
152
if ie.kind == "symlink":
153
ie.symlink_target = blob.data
157
ie.text_size = len(blob.data)
158
ie.text_sha1 = osutils.sha_string(blob.data)
159
# Check what revision we should store
161
for pinv in parent_invs:
162
if not file_id in pinv:
164
if pinv[file_id].text_sha1 == ie.text_sha1:
165
# found a revision in one of the parents to use
166
ie.revision = pinv[file_id].revision
168
parent_keys.append((file_id, pinv[file_id].revision))
169
if ie.revision is None:
170
# Need to store a new revision
171
ie.revision = revision_id
172
assert file_id is not None
173
assert ie.revision is not None
174
texts.add_lines((file_id, ie.revision), parent_keys,
175
osutils.split_lines(blob.data))
176
if "verify" in debug.debug_flags:
177
assert text_to_blob(blob.data).id == hexsha
178
shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))
179
if file_id in base_inv:
180
old_path = base_inv.id2path(file_id)
183
return [(old_path, path, file_id, ie)]
186
def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,
187
revision_id, parent_invs, shagitmap, lookup_object):
82
188
"""Import a git tree object into a bzr repository.
84
:param repo: A Bzr repository object
190
:param texts: VersionedFiles object to add to
85
191
:param path: Path in the tree
86
192
:param tree: A git tree object
87
:param inv: Inventory object
193
:param base_inv: Base inventory against which to return inventory delta
194
:return: Inventory delta for this subtree
89
197
file_id = mapping.generate_file_id(path)
90
repo.texts.add_lines((file_id, tree.id),
93
inv.add_path(path, "directory", file_id)
198
# We just have to hope this is indeed utf-8:
199
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
201
if not file_id in base_inv:
202
# Newly appeared here
203
ie.revision = revision_id
204
texts.add_lines((file_id, ie.revision), [], [])
205
invdelta.append((None, path, file_id, ie))
207
# See if this has changed at all
209
base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)
213
if base_sha == hexsha:
214
# If nothing has changed since the base revision, we're done
216
# Remember for next time
217
existing_children = set()
218
if "verify" in debug.debug_flags:
221
shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))
223
tree = lookup_object(hexsha)
94
224
for mode, name, hexsha in tree.entries():
95
225
entry_kind = (mode & 0700000) / 0100000
96
226
basename = name.decode("utf-8")
100
child_path = urlutils.join(path, name)
227
existing_children.add(basename)
228
child_path = osutils.pathjoin(path, name)
101
229
if entry_kind == 0:
102
import_git_tree(repo, mapping, child_path, lookup_object, inv)
230
if mode != DEFAULT_TREE_MODE:
231
child_modes[child_path] = mode
232
subinvdelta, grandchildmodes = import_git_tree(texts, mapping, child_path, hexsha,
233
base_inv, file_id, revision_id, parent_invs, shagitmap,
235
invdelta.extend(subinvdelta)
236
child_modes.update(grandchildmodes)
103
237
elif entry_kind == 1:
104
import_git_blob(repo, mapping, child_path, lookup_object, inv)
238
fs_mode = mode & 0777
239
file_kind = (mode & 070000) / 010000
240
if file_kind == 0: # regular file
242
if mode not in (DEFAULT_FILE_MODE, DEFAULT_FILE_MODE|0111):
243
child_modes[child_path] = mode
246
if mode != DEFAULT_SYMLINK_MODE:
247
child_modes[child_path] = mode
249
raise AssertionError("Unknown file kind, mode=%r" % (mode,))
250
subinvdelta = import_git_blob(texts, mapping, child_path, hexsha,
251
base_inv, file_id, revision_id, parent_invs, shagitmap,
252
lookup_object, bool(fs_mode & 0111), symlink)
253
invdelta.extend(subinvdelta)
106
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
109
def import_git_objects(repo, mapping, object_iter):
255
raise AssertionError("Unknown object kind, perms=%r." % (mode,))
256
# Remove any children that have disappeared
257
if file_id in base_inv:
258
deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]
261
invdelta.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))
262
if ie.kind == "directory":
263
deletable.extend(ie.children.values())
264
return invdelta, child_modes
267
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
110
269
"""Import a set of git objects into a bzr repository.
112
271
:param repo: Bazaar repository
114
273
:param object_iter: Iterator over Git objects.
116
275
# TODO: a more (memory-)efficient implementation of this
118
for o in object_iter:
281
parent_invs_cache = LRUCache(50)
121
282
# Find and convert commit objects
122
for o in objects.iterkeys():
285
pb.update("finding revisions to fetch", len(graph), None)
287
assert isinstance(head, str)
289
o = object_iter[head]
123
292
if isinstance(o, Commit):
124
293
rev = mapping.import_commit(o)
125
root_trees[rev] = objects[o.tree_sha]
294
if repo.has_revision(rev.revision_id):
296
root_trees[rev.revision_id] = o.tree
297
revisions[rev.revision_id] = rev
298
graph.append((rev.revision_id, rev.parent_ids))
299
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),
300
"commit", (rev.revision_id, o._tree))
301
heads.extend([p for p in o.parents if p not in checked])
302
elif isinstance(o, Tag):
303
heads.append(o.object[1])
305
trace.warning("Unable to import head object %r" % o)
307
# Order the revisions
126
308
# Create the inventory objects
127
for rev, root_tree in root_trees.iteritems():
309
for i, revid in enumerate(topo_sort(graph)):
311
pb.update("fetching revisions", i, len(graph))
312
rev = revisions[revid]
128
313
# We have to do this here, since we have to walk the tree and
129
# we need to make sure to import the blobs / trees with the riht
314
# we need to make sure to import the blobs / trees with the right
130
315
# path; this may involve adding them more than once.
132
316
def lookup_object(sha):
135
return reconstruct_git_object(repo, mapping, sha)
136
import_git_tree(repo, mapping, "", tree, inv, lookup_object)
137
repo.add_revision(rev.revision_id, rev, inv)
140
def reconstruct_git_commit(repo, rev):
141
raise NotImplementedError(self.reconstruct_git_commit)
144
def reconstruct_git_object(repo, mapping, sha):
146
revid = mapping.revision_id_foreign_to_bzr(sha)
148
rev = repo.get_revision(revid)
149
except NoSuchRevision:
152
return reconstruct_git_commit(rev)
156
raise KeyError("No such object %s" % sha)
159
class InterGitRepository(InterRepository):
161
_matching_repo_format = GitFormat()
318
return object_iter[sha]
320
return target_git_object_retriever[sha]
322
for parent_id in rev.parent_ids:
324
parent_invs.append(parent_invs_cache[parent_id])
326
parent_inv = repo.get_inventory(parent_id)
327
parent_invs.append(parent_inv)
328
parent_invs_cache[parent_id] = parent_inv
329
if parent_invs == []:
330
base_inv = Inventory(root_id=None)
332
base_inv = parent_invs[0]
333
inv_delta, unusual_modes = import_git_tree(repo.texts, mapping, "",
334
root_trees[revid], base_inv, None, revid, parent_invs,
335
target_git_object_retriever._idmap, lookup_object)
336
if unusual_modes != {}:
337
ret = "unusual modes: \n"
338
for item in unusual_modes.iteritems():
339
ret += "\t%s: %o\n" % item
340
raise AssertionError(ret)
342
basis_id = rev.parent_ids[0]
344
basis_id = NULL_REVISION
345
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
346
inv_delta, rev.revision_id, rev.parent_ids)
347
parent_invs_cache[rev.revision_id] = inv
348
repo.add_revision(rev.revision_id, rev)
349
target_git_object_retriever._idmap.commit()
352
class InterGitNonGitRepository(InterRepository):
353
"""Base InterRepository that copies revisions from a Git into a non-Git
356
_matching_repo_format = GitRepositoryFormat()
164
359
def _get_repo_format_to_test():
168
363
"""See InterRepository.copy_content."""
169
364
self.fetch(revision_id, pb, find_ghosts=False)
171
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
366
def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,
368
self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,
369
mapping=mapping, fetch_spec=fetch_spec)
371
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
372
mapping=None, fetch_spec=None):
173
373
if mapping is None:
174
374
mapping = self.source.get_mapping()
177
pb.note("git: %s" % text)
179
info("git: %s" % text)
180
def determine_wants(heads):
181
if revision_id is None:
184
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
375
if revision_id is not None:
376
interesting_heads = [revision_id]
377
elif fetch_spec is not None:
378
interesting_heads = fetch_spec.heads
380
interesting_heads = None
382
def determine_wants(refs):
384
if interesting_heads is None:
385
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
387
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid != NULL_REVISION]
185
388
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
389
self.fetch_objects(determine_wants, mapping, pb)
394
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
395
"""InterRepository that copies revisions from a remote Git into a non-Git
398
def fetch_objects(self, determine_wants, mapping, pb=None):
400
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
186
401
graph_walker = BzrFetchGraphWalker(self.target, mapping)
187
self.target.lock_write()
189
import_git_objects(self.target, mapping,
190
self.source.fetch_objects(determine_wants, graph_walker,
404
create_pb = pb = ui.ui_factory.nested_progress_bar()
405
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
408
def record_determine_wants(heads):
409
wants = determine_wants(heads)
410
recorded_wants.extend(wants)
414
self.target.lock_write()
416
self.target.start_write_group()
418
objects_iter = self.source.fetch_objects(
419
record_determine_wants,
421
target_git_object_retriever.get_raw,
423
import_git_objects(self.target, mapping, objects_iter,
424
target_git_object_retriever, recorded_wants, pb)
426
self.target.commit_write_group()
434
def is_compatible(source, target):
435
"""Be compatible with GitRepository."""
436
# FIXME: Also check target uses VersionedFile
437
return (isinstance(source, RemoteGitRepository) and
438
target.supports_rich_root() and
439
not isinstance(target, GitRepository))
442
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
443
"""InterRepository that copies revisions from a remote Git into a non-Git
446
def fetch_objects(self, determine_wants, mapping, pb=None):
447
wants = determine_wants(self.source._git.get_refs())
450
create_pb = pb = ui.ui_factory.nested_progress_bar()
451
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
453
self.target.lock_write()
455
self.target.start_write_group()
457
import_git_objects(self.target, mapping,
458
self.source._git.object_store,
459
target_git_object_retriever, wants, pb)
461
self.target.commit_write_group()
196
469
def is_compatible(source, target):
197
470
"""Be compatible with GitRepository."""
198
471
# FIXME: Also check target uses VersionedFile
199
472
return (isinstance(source, LocalGitRepository) and
200
target.supports_rich_root())
473
target.supports_rich_root() and
474
not isinstance(target, GitRepository))
477
class InterGitRepository(InterRepository):
478
"""InterRepository that copies between Git repositories."""
480
_matching_repo_format = GitRepositoryFormat()
483
def _get_repo_format_to_test():
486
def copy_content(self, revision_id=None, pb=None):
487
"""See InterRepository.copy_content."""
488
self.fetch(revision_id, pb, find_ghosts=False)
490
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
491
mapping=None, fetch_spec=None):
493
mapping = self.source.get_mapping()
495
trace.info("git: %s", text)
497
if revision_id is not None:
498
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
499
elif fetch_spec is not None:
500
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
501
if fetch_spec is None and revision_id is None:
502
determine_wants = r.object_store.determine_wants_all
504
determine_wants = lambda x: [y for y in args if not y in r.object_store]
506
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
507
f, commit = r.object_store.add_thin_pack()
509
self.source.fetch_pack(determine_wants, graphwalker, f.write, progress)
516
def is_compatible(source, target):
517
"""Be compatible with GitRepository."""
518
return (isinstance(source, GitRepository) and
519
isinstance(target, GitRepository))