14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import (
21
from dulwich.client import (
22
SimpleFetchGraphWalker,
24
from dulwich.objects import (
35
from bzrlib.errors import (
39
from bzrlib.inventory import (
45
from bzrlib.lru_cache import (
48
from bzrlib.repository import (
51
from bzrlib.revision import (
54
from bzrlib.tsort import (
58
from bzrlib.plugins.git.converter import (
61
from bzrlib.plugins.git.repository import (
66
from bzrlib.plugins.git.remote import (
17
from bzrlib import osutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
23
from bzrlib.plugins.git import git
24
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
25
from bzrlib.plugins.git.remote import RemoteGitRepository
27
from dulwich.objects import Commit
29
from cStringIO import StringIO
71
32
class BzrFetchGraphWalker(object):
72
"""GraphWalker implementation that uses a Bazaar repository."""
74
34
def __init__(self, repository, mapping):
75
35
self.repository = repository
101
58
self.heads.update([p for p in ps if not p in self.done])
103
60
self.done.add(ret)
104
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
61
return self.mapping.revision_id_bzr_to_foreign(ret)
105
62
except InvalidRevisionId:
110
def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,
111
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
67
def import_git_blob(repo, mapping, path, blob):
112
68
"""Import a git blob object into a bzr repository.
114
:param texts: VersionedFiles to add to
70
:param repo: bzr repository
115
71
:param path: Path in the tree
116
72
:param blob: A git blob
117
:return: Inventory delta for this file
119
74
file_id = mapping.generate_file_id(path)
124
# We just have to hope this is indeed utf-8:
125
ie = cls(file_id, urlutils.basename(path).decode("utf-8"),
127
ie.executable = executable
128
# See if this has changed at all
130
base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)
134
if (base_sha == hexsha and base_inv[file_id].executable == ie.executable
135
and base_inv[file_id].kind == ie.kind):
136
# If nothing has changed since the base revision, we're done
138
if base_sha == hexsha:
139
ie.text_size = base_inv[file_id].text_size
140
ie.text_sha1 = base_inv[file_id].text_sha1
141
ie.symlink_target = base_inv[file_id].symlink_target
142
ie.revision = base_inv[file_id].revision
144
blob = lookup_object(hexsha)
145
if ie.kind == "symlink":
146
ie.symlink_target = blob.data
150
ie.text_size = len(blob.data)
151
ie.text_sha1 = osutils.sha_string(blob.data)
152
# Check what revision we should store
154
for pinv in parent_invs:
155
if not file_id in pinv:
157
if pinv[file_id].text_sha1 == ie.text_sha1:
158
# found a revision in one of the parents to use
159
ie.revision = pinv[file_id].revision
161
parent_keys.append((file_id, pinv[file_id].revision))
162
if ie.revision is None:
163
# Need to store a new revision
164
ie.revision = revision_id
165
assert file_id is not None
166
assert ie.revision is not None
167
texts.add_lines((file_id, ie.revision), parent_keys,
168
osutils.split_lines(blob.data))
169
shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))
170
if file_id in base_inv:
171
old_path = base_inv.id2path(file_id)
174
return [(old_path, path, file_id, ie)]
177
def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,
178
revision_id, parent_invs, shagitmap, lookup_object):
75
repo.texts.add_lines((file_id, blob.id),
77
osutils.split_lines(blob.data))
78
inv.add_path(path, "file", file_id)
81
def import_git_tree(repo, mapping, path, tree, inv, lookup_object):
179
82
"""Import a git tree object into a bzr repository.
181
:param texts: VersionedFiles object to add to
84
:param repo: A Bzr repository object
182
85
:param path: Path in the tree
183
86
:param tree: A git tree object
184
:param base_inv: Base inventory against which to return inventory delta
185
:return: Inventory delta for this subtree
87
:param inv: Inventory object
188
89
file_id = mapping.generate_file_id(path)
189
# We just have to hope this is indeed utf-8:
190
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
192
if not file_id in base_inv:
193
# Newly appeared here
194
ie.revision = revision_id
195
texts.add_lines((file_id, ie.revision), [], [])
196
ret.append((None, path, file_id, ie))
198
# See if this has changed at all
200
base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)
204
if base_sha == hexsha:
205
# If nothing has changed since the base revision, we're done
207
# Remember for next time
208
existing_children = set()
209
shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))
210
tree = lookup_object(hexsha)
90
repo.texts.add_lines((file_id, tree.id),
93
inv.add_path(path, "directory", file_id)
211
94
for mode, name, hexsha in tree.entries():
212
95
entry_kind = (mode & 0700000) / 0100000
213
96
basename = name.decode("utf-8")
214
existing_children.add(basename)
218
100
child_path = urlutils.join(path, name)
219
101
if entry_kind == 0:
220
ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,
221
file_id, revision_id, parent_invs, shagitmap, lookup_object))
102
import_git_tree(repo, mapping, child_path, lookup_object, inv)
222
103
elif entry_kind == 1:
223
fs_mode = mode & 0777
224
file_kind = (mode & 070000) / 010000
225
if file_kind == 0: # regular file
230
raise AssertionError("Unknown file kind, mode=%r" % (mode,))
231
ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,
232
file_id, revision_id, parent_invs, shagitmap, lookup_object,
233
bool(fs_mode & 0111), symlink))
104
import_git_blob(repo, mapping, child_path, lookup_object, inv)
235
raise AssertionError("Unknown object kind, perms=%r." % (mode,))
236
# Remove any children that have disappeared
237
if file_id in base_inv:
238
deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]
241
ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))
242
if ie.kind == "directory":
243
deletable.extend(ie.children.values())
247
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
106
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
109
def import_git_objects(repo, mapping, object_iter):
249
110
"""Import a set of git objects into a bzr repository.
251
112
:param repo: Bazaar repository
253
114
:param object_iter: Iterator over Git objects.
255
116
# TODO: a more (memory-)efficient implementation of this
118
for o in object_iter:
261
parent_invs_cache = LRUCache(50)
262
121
# Find and convert commit objects
265
pb.update("finding revisions to fetch", len(graph), None)
267
assert isinstance(head, str)
269
o = object_iter[head]
122
for o in objects.iterkeys():
272
123
if isinstance(o, Commit):
273
124
rev = mapping.import_commit(o)
274
if repo.has_revision(rev.revision_id):
276
root_trees[rev.revision_id] = o.tree
277
revisions[rev.revision_id] = rev
278
graph.append((rev.revision_id, rev.parent_ids))
279
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),
280
"commit", (rev.revision_id, o._tree))
281
heads.extend([p for p in o.parents if p not in checked])
282
elif isinstance(o, Tag):
283
heads.append(o.object[1])
285
trace.warning("Unable to import head object %r" % o)
287
# Order the revisions
125
root_trees[rev] = objects[o.tree_sha]
288
126
# Create the inventory objects
289
for i, revid in enumerate(topo_sort(graph)):
291
pb.update("fetching revisions", i, len(graph))
292
rev = revisions[revid]
127
for rev, root_tree in root_trees.iteritems():
293
128
# We have to do this here, since we have to walk the tree and
294
# we need to make sure to import the blobs / trees with the right
129
# we need to make sure to import the blobs / trees with the riht
295
130
# path; this may involve adding them more than once.
296
132
def lookup_object(sha):
298
return object_iter[sha]
300
return target_git_object_retriever[sha]
302
for parent_id in rev.parent_ids:
304
parent_invs.append(parent_invs_cache[parent_id])
306
parent_inv = repo.get_inventory(parent_id)
307
parent_invs.append(parent_inv)
308
parent_invs_cache[parent_id] = parent_inv
309
if parent_invs == []:
310
base_inv = Inventory(root_id=None)
312
base_inv = parent_invs[0]
313
inv_delta = import_git_tree(repo.texts, mapping, "",
314
root_trees[revid], base_inv, None, revid, parent_invs,
315
target_git_object_retriever._idmap, lookup_object)
317
basis_id = rev.parent_ids[0]
319
basis_id = NULL_REVISION
320
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
321
inv_delta, rev.revision_id, rev.parent_ids)
322
parent_invs_cache[rev.revision_id] = inv
323
repo.add_revision(rev.revision_id, rev)
324
target_git_object_retriever._idmap.commit()
327
class InterGitNonGitRepository(InterRepository):
328
"""Base InterRepository that copies revisions from a Git into a non-Git
331
_matching_repo_format = GitRepositoryFormat()
135
return reconstruct_git_object(repo, mapping, sha)
136
import_git_tree(repo, mapping, "", tree, inv, lookup_object)
137
repo.add_revision(rev.revision_id, rev, inv)
140
def reconstruct_git_commit(repo, rev):
141
raise NotImplementedError(self.reconstruct_git_commit)
144
def reconstruct_git_object(repo, mapping, sha):
146
revid = mapping.revision_id_foreign_to_bzr(sha)
148
rev = repo.get_revision(revid)
149
except NoSuchRevision:
152
return reconstruct_git_commit(rev)
156
raise KeyError("No such object %s" % sha)
159
class InterGitRepository(InterRepository):
161
_matching_repo_format = GitFormat()
334
164
def _get_repo_format_to_test():
338
168
"""See InterRepository.copy_content."""
339
169
self.fetch(revision_id, pb, find_ghosts=False)
341
def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,
343
self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,
344
mapping=mapping, fetch_spec=fetch_spec)
346
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
347
mapping=None, fetch_spec=None):
171
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
348
173
if mapping is None:
349
174
mapping = self.source.get_mapping()
350
if revision_id is not None:
351
interesting_heads = [revision_id]
352
elif fetch_spec is not None:
353
interesting_heads = fetch_spec.heads
355
interesting_heads = None
357
def determine_wants(refs):
359
if interesting_heads is None:
360
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
362
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]
177
pb.note("git: %s" % text)
179
info("git: %s" % text)
180
def determine_wants(heads):
181
if revision_id is None:
184
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
363
185
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
364
self.fetch_objects(determine_wants, mapping, pb)
369
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
370
"""InterRepository that copies revisions from a remote Git into a non-Git
373
def fetch_objects(self, determine_wants, mapping, pb=None):
375
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
376
186
graph_walker = BzrFetchGraphWalker(self.target, mapping)
379
create_pb = pb = ui.ui_factory.nested_progress_bar()
380
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
383
def record_determine_wants(heads):
384
wants = determine_wants(heads)
385
recorded_wants.extend(wants)
389
self.target.lock_write()
391
self.target.start_write_group()
393
objects_iter = self.source.fetch_objects(
394
record_determine_wants,
396
target_git_object_retriever.get_raw,
398
import_git_objects(self.target, mapping, objects_iter,
399
target_git_object_retriever, recorded_wants, pb)
401
self.target.commit_write_group()
409
def is_compatible(source, target):
410
"""Be compatible with GitRepository."""
411
# FIXME: Also check target uses VersionedFile
412
return (isinstance(source, RemoteGitRepository) and
413
target.supports_rich_root() and
414
not isinstance(target, GitRepository))
417
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
418
"""InterRepository that copies revisions from a remote Git into a non-Git
421
def fetch_objects(self, determine_wants, mapping, pb=None):
422
wants = determine_wants(self.source._git.get_refs())
425
create_pb = pb = ui.ui_factory.nested_progress_bar()
426
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
428
self.target.lock_write()
430
self.target.start_write_group()
432
import_git_objects(self.target, mapping,
433
self.source._git.object_store,
434
target_git_object_retriever, wants, pb)
436
self.target.commit_write_group()
187
self.target.lock_write()
189
import_git_objects(self.target, mapping,
190
self.source.fetch_objects(determine_wants, graph_walker,
444
196
def is_compatible(source, target):
445
197
"""Be compatible with GitRepository."""
446
198
# FIXME: Also check target uses VersionedFile
447
199
return (isinstance(source, LocalGitRepository) and
448
target.supports_rich_root() and
449
not isinstance(target, GitRepository))
452
class InterGitRepository(InterRepository):
453
"""InterRepository that copies between Git repositories."""
455
_matching_repo_format = GitRepositoryFormat()
458
def _get_repo_format_to_test():
461
def copy_content(self, revision_id=None, pb=None):
462
"""See InterRepository.copy_content."""
463
self.fetch(revision_id, pb, find_ghosts=False)
465
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
466
mapping=None, fetch_spec=None):
468
mapping = self.source.get_mapping()
470
trace.info("git: %s", text)
472
if revision_id is not None:
473
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
474
elif fetch_spec is not None:
475
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
476
if fetch_spec is None and revision_id is None:
477
determine_wants = r.object_store.determine_wants_all
479
determine_wants = lambda x: [y for y in args if not y in r.object_store]
481
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
482
f, commit = r.object_store.add_pack()
484
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
492
def is_compatible(source, target):
493
"""Be compatible with GitRepository."""
494
return (isinstance(source, GitRepository) and
495
isinstance(target, GitRepository))
200
target.supports_rich_root())
added
removed
fetch.py
