14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import (
21
from dulwich.client import (
22
SimpleFetchGraphWalker,
24
from dulwich.objects import (
35
from bzrlib.errors import (
39
from bzrlib.inventory import (
45
from bzrlib.lru_cache import (
48
from bzrlib.repository import (
51
from bzrlib.revision import (
54
from bzrlib.tsort import (
58
from bzrlib.plugins.git.converter import (
61
from bzrlib.plugins.git.repository import (
66
from bzrlib.plugins.git.remote import (
17
from bzrlib import osutils, urlutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
23
from bzrlib.plugins.git import git
24
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
25
from bzrlib.plugins.git.remote import RemoteGitRepository
27
from dulwich.objects import Commit
29
from cStringIO import StringIO
71
32
class BzrFetchGraphWalker(object):
72
"""GraphWalker implementation that uses a Bazaar repository."""
74
34
def __init__(self, repository, mapping):
75
35
self.repository = repository
101
58
self.heads.update([p for p in ps if not p in self.done])
103
60
self.done.add(ret)
104
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
61
return self.mapping.revision_id_bzr_to_foreign(ret)
105
62
except InvalidRevisionId:
110
def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,
111
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
67
def import_git_blob(repo, mapping, path, blob, inv):
112
68
"""Import a git blob object into a bzr repository.
114
:param texts: VersionedFiles to add to
70
:param repo: bzr repository
115
71
:param path: Path in the tree
116
72
:param blob: A git blob
117
:return: Inventory delta for this file
119
74
file_id = mapping.generate_file_id(path)
124
# We just have to hope this is indeed utf-8:
125
ie = cls(file_id, urlutils.basename(path).decode("utf-8"),
127
ie.executable = executable
128
# See if this has changed at all
130
base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)
134
if (base_sha == hexsha and base_inv[file_id].executable == ie.executable
135
and base_inv[file_id].kind == ie.kind):
136
# If nothing has changed since the base revision, we're done
138
if base_sha == hexsha:
139
ie.text_size = base_inv[file_id].text_size
140
ie.text_sha1 = base_inv[file_id].text_sha1
141
ie.symlink_target = base_inv[file_id].symlink_target
142
ie.revision = base_inv[file_id].revision
144
blob = lookup_object(hexsha)
145
if ie.kind == "symlink":
146
ie.symlink_target = blob.data
150
ie.text_size = len(blob.data)
151
ie.text_sha1 = osutils.sha_string(blob.data)
152
# Check what revision we should store
154
for pinv in parent_invs:
155
if not file_id in pinv:
157
if pinv[file_id].text_sha1 == ie.text_sha1:
158
# found a revision in one of the parents to use
159
ie.revision = pinv[file_id].revision
161
parent_keys.append((file_id, pinv[file_id].revision))
162
if ie.revision is None:
163
# Need to store a new revision
164
ie.revision = revision_id
165
assert file_id is not None
166
assert ie.revision is not None
167
texts.add_lines((file_id, ie.revision), parent_keys,
168
osutils.split_lines(blob.data))
169
shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))
170
if file_id in base_inv:
171
old_path = base_inv.id2path(file_id)
174
return [(old_path, path, file_id, ie)]
177
def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,
178
revision_id, parent_invs, shagitmap, lookup_object):
75
repo.texts.add_lines((file_id, blob.id),
77
osutils.split_lines(blob.data))
78
ie = inv.add_path(path, "file", file_id)
81
def import_git_tree(repo, mapping, path, tree, inv, lookup_object):
179
82
"""Import a git tree object into a bzr repository.
181
:param texts: VersionedFiles object to add to
84
:param repo: A Bzr repository object
182
85
:param path: Path in the tree
183
86
:param tree: A git tree object
184
:param base_inv: Base inventory against which to return inventory delta
185
:return: Inventory delta for this subtree
87
:param inv: Inventory object
188
89
file_id = mapping.generate_file_id(path)
189
# We just have to hope this is indeed utf-8:
190
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
192
if not file_id in base_inv:
193
# Newly appeared here
194
ie.revision = revision_id
195
texts.add_lines((file_id, ie.revision), [], [])
196
ret.append((None, path, file_id, ie))
198
# See if this has changed at all
200
base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)
204
if base_sha == hexsha:
205
# If nothing has changed since the base revision, we're done
207
# Remember for next time
208
existing_children = set()
209
shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))
210
tree = lookup_object(hexsha)
90
repo.texts.add_lines((file_id, tree.id),
93
inv.add_path(path, "directory", file_id)
211
94
for mode, name, hexsha in tree.entries():
212
95
entry_kind = (mode & 0700000) / 0100000
213
96
basename = name.decode("utf-8")
214
existing_children.add(basename)
218
100
child_path = urlutils.join(path, name)
219
101
if entry_kind == 0:
220
ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,
221
file_id, revision_id, parent_invs, shagitmap, lookup_object))
102
tree = lookup_object(hexsha)
103
import_git_tree(repo, mapping, child_path, tree, inv, lookup_object)
222
104
elif entry_kind == 1:
223
fs_mode = mode & 0777
224
file_kind = (mode & 070000) / 010000
225
if file_kind == 0: # regular file
230
raise AssertionError("Unknown file kind, mode=%r" % (mode,))
231
ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,
232
file_id, revision_id, parent_invs, shagitmap, lookup_object,
233
bool(fs_mode & 0111), symlink))
105
blob = lookup_object(hexsha)
106
import_git_blob(repo, mapping, child_path, blob, inv)
235
raise AssertionError("Unknown object kind, perms=%r." % (mode,))
236
# Remove any children that have disappeared
237
if file_id in base_inv:
238
deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]
241
ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))
242
if ie.kind == "directory":
243
deletable.extend(ie.children.values())
247
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
108
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
111
def import_git_objects(repo, mapping, object_iter):
249
112
"""Import a set of git objects into a bzr repository.
251
114
:param repo: Bazaar repository
253
116
:param object_iter: Iterator over Git objects.
255
118
# TODO: a more (memory-)efficient implementation of this
120
for o in object_iter:
261
parent_invs_cache = LRUCache(50)
262
123
# Find and convert commit objects
265
pb.update("finding revisions to fetch", len(graph), None)
267
assert isinstance(head, str)
269
o = object_iter[head]
124
for o in objects.itervalues():
272
125
if isinstance(o, Commit):
273
126
rev = mapping.import_commit(o)
274
if repo.has_revision(rev.revision_id):
276
root_trees[rev.revision_id] = o.tree
277
revisions[rev.revision_id] = rev
278
graph.append((rev.revision_id, rev.parent_ids))
279
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),
280
"commit", (rev.revision_id, o._tree))
281
heads.extend([p for p in o.parents if p not in checked])
282
elif isinstance(o, Tag):
283
heads.append(o.object[1])
285
trace.warning("Unable to import head object %r" % o)
287
# Order the revisions
127
root_trees[rev] = objects[o.tree]
288
128
# Create the inventory objects
289
for i, revid in enumerate(topo_sort(graph)):
291
pb.update("fetching revisions", i, len(graph))
292
rev = revisions[revid]
129
for rev, root_tree in root_trees.iteritems():
293
130
# We have to do this here, since we have to walk the tree and
294
# we need to make sure to import the blobs / trees with the right
131
# we need to make sure to import the blobs / trees with the riht
295
132
# path; this may involve adding them more than once.
134
inv.revision_id = rev.revision_id
296
135
def lookup_object(sha):
298
return object_iter[sha]
300
return target_git_object_retriever[sha]
302
for parent_id in rev.parent_ids:
304
parent_invs.append(parent_invs_cache[parent_id])
306
parent_inv = repo.get_inventory(parent_id)
307
parent_invs.append(parent_inv)
308
parent_invs_cache[parent_id] = parent_inv
309
if parent_invs == []:
310
base_inv = Inventory(root_id=None)
312
base_inv = parent_invs[0]
313
inv_delta = import_git_tree(repo.texts, mapping, "",
314
root_trees[revid], base_inv, None, revid, parent_invs,
315
target_git_object_retriever._idmap, lookup_object)
317
basis_id = rev.parent_ids[0]
319
basis_id = NULL_REVISION
320
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
321
inv_delta, rev.revision_id, rev.parent_ids)
322
parent_invs_cache[rev.revision_id] = inv
323
repo.add_revision(rev.revision_id, rev)
324
target_git_object_retriever._idmap.commit()
327
class InterGitNonGitRepository(InterRepository):
328
"""Base InterRepository that copies revisions from a Git into a non-Git
331
_matching_repo_format = GitRepositoryFormat()
138
return reconstruct_git_object(repo, mapping, sha)
139
import_git_tree(repo, mapping, "", root_tree, inv, lookup_object)
140
repo.add_revision(rev.revision_id, rev, inv)
143
def reconstruct_git_commit(repo, rev):
144
raise NotImplementedError(self.reconstruct_git_commit)
147
def reconstruct_git_object(repo, mapping, sha):
149
revid = mapping.revision_id_foreign_to_bzr(sha)
151
rev = repo.get_revision(revid)
152
except NoSuchRevision:
155
return reconstruct_git_commit(rev)
159
raise KeyError("No such object %s" % sha)
162
class InterGitRepository(InterRepository):
164
_matching_repo_format = GitFormat()
334
167
def _get_repo_format_to_test():
338
171
"""See InterRepository.copy_content."""
339
172
self.fetch(revision_id, pb, find_ghosts=False)
341
def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,
343
self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,
344
mapping=mapping, fetch_spec=fetch_spec)
346
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
347
mapping=None, fetch_spec=None):
174
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
348
176
if mapping is None:
349
177
mapping = self.source.get_mapping()
350
if revision_id is not None:
351
interesting_heads = [revision_id]
352
elif fetch_spec is not None:
353
interesting_heads = fetch_spec.heads
355
interesting_heads = None
357
def determine_wants(refs):
359
if interesting_heads is None:
360
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
362
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]
180
pb.note("git: %s" % text)
182
info("git: %s" % text)
183
def determine_wants(heads):
184
if revision_id is None:
187
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
363
188
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
364
self.fetch_objects(determine_wants, mapping, pb)
369
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
370
"""InterRepository that copies revisions from a remote Git into a non-Git
373
def fetch_objects(self, determine_wants, mapping, pb=None):
375
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
376
189
graph_walker = BzrFetchGraphWalker(self.target, mapping)
379
create_pb = pb = ui.ui_factory.nested_progress_bar()
380
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
383
def record_determine_wants(heads):
384
wants = determine_wants(heads)
385
recorded_wants.extend(wants)
389
self.target.lock_write()
391
self.target.start_write_group()
393
objects_iter = self.source.fetch_objects(
394
record_determine_wants,
396
target_git_object_retriever.get_raw,
398
import_git_objects(self.target, mapping, objects_iter,
399
target_git_object_retriever, recorded_wants, pb)
401
self.target.commit_write_group()
409
def is_compatible(source, target):
410
"""Be compatible with GitRepository."""
411
# FIXME: Also check target uses VersionedFile
412
return (isinstance(source, RemoteGitRepository) and
413
target.supports_rich_root() and
414
not isinstance(target, GitRepository))
417
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
418
"""InterRepository that copies revisions from a remote Git into a non-Git
421
def fetch_objects(self, determine_wants, mapping, pb=None):
422
wants = determine_wants(self.source._git.get_refs())
425
create_pb = pb = ui.ui_factory.nested_progress_bar()
426
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
428
self.target.lock_write()
430
self.target.start_write_group()
432
import_git_objects(self.target, mapping,
433
self.source._git.object_store,
434
target_git_object_retriever, wants, pb)
436
self.target.commit_write_group()
190
self.target.lock_write()
192
self.target.start_write_group()
194
import_git_objects(self.target, mapping,
195
iter(self.source.fetch_objects(determine_wants, graph_walker,
198
self.target.commit_write_group()
444
203
def is_compatible(source, target):
445
204
"""Be compatible with GitRepository."""
446
205
# FIXME: Also check target uses VersionedFile
447
206
return (isinstance(source, LocalGitRepository) and
448
target.supports_rich_root() and
449
not isinstance(target, GitRepository))
452
class InterGitRepository(InterRepository):
453
"""InterRepository that copies between Git repositories."""
455
_matching_repo_format = GitRepositoryFormat()
458
def _get_repo_format_to_test():
461
def copy_content(self, revision_id=None, pb=None):
462
"""See InterRepository.copy_content."""
463
self.fetch(revision_id, pb, find_ghosts=False)
465
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
466
mapping=None, fetch_spec=None):
468
mapping = self.source.get_mapping()
470
trace.info("git: %s", text)
472
if revision_id is not None:
473
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
474
elif fetch_spec is not None:
475
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
476
if fetch_spec is None and revision_id is None:
477
determine_wants = r.object_store.determine_wants_all
479
determine_wants = lambda x: [y for y in args if not y in r.object_store]
481
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
482
f, commit = r.object_store.add_pack()
484
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
492
def is_compatible(source, target):
493
"""Be compatible with GitRepository."""
494
return (isinstance(source, GitRepository) and
495
isinstance(target, GitRepository))
207
target.supports_rich_root())
added
removed
fetch.py
