14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import (
21
from dulwich.client import (
22
SimpleFetchGraphWalker,
24
from dulwich.objects import (
35
from bzrlib.errors import (
39
from bzrlib.inventory import (
45
from bzrlib.lru_cache import (
48
from bzrlib.repository import (
51
from bzrlib.revision import (
54
from bzrlib.tsort import (
58
from bzrlib.plugins.git.converter import (
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
24
from bzrlib.plugins.git import git
61
25
from bzrlib.plugins.git.repository import (
66
from bzrlib.plugins.git.remote import (
30
from bzrlib.plugins.git.remote import RemoteGitRepository
32
from dulwich.client import SimpleFetchGraphWalker
33
from dulwich.objects import Commit
35
from cStringIO import StringIO
71
38
class BzrFetchGraphWalker(object):
72
"""GraphWalker implementation that uses a Bazaar repository."""
74
40
def __init__(self, repository, mapping):
75
41
self.repository = repository
101
64
self.heads.update([p for p in ps if not p in self.done])
103
66
self.done.add(ret)
104
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
67
return self.mapping.revision_id_bzr_to_foreign(ret)
105
68
except InvalidRevisionId:
110
def import_git_blob(texts, mapping, path, hexsha, base_inv, parent_id,
111
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
73
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, executable):
112
74
"""Import a git blob object into a bzr repository.
114
:param texts: VersionedFiles to add to
76
:param repo: bzr repository
115
77
:param path: Path in the tree
116
78
:param blob: A git blob
117
:return: Inventory delta for this file
119
80
file_id = mapping.generate_file_id(path)
124
# We just have to hope this is indeed utf-8:
125
ie = cls(file_id, urlutils.basename(path).decode("utf-8"),
81
text_revision = inv.revision_id
82
repo.texts.add_lines((file_id, text_revision),
83
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
84
osutils.split_lines(blob.data))
85
ie = inv.add_path(path, "file", file_id)
86
ie.revision = text_revision
87
ie.text_size = len(blob.data)
88
ie.text_sha1 = osutils.sha_string(blob.data)
127
89
ie.executable = executable
128
# See if this has changed at all
130
base_sha = shagitmap.lookup_blob(file_id, base_inv.revision_id)
134
if (base_sha == hexsha and base_inv[file_id].executable == ie.executable
135
and base_inv[file_id].kind == ie.kind):
136
# If nothing has changed since the base revision, we're done
138
if base_sha == hexsha:
139
ie.text_size = base_inv[file_id].text_size
140
ie.text_sha1 = base_inv[file_id].text_sha1
141
ie.symlink_target = base_inv[file_id].symlink_target
142
ie.revision = base_inv[file_id].revision
144
blob = lookup_object(hexsha)
145
if ie.kind == "symlink":
146
ie.symlink_target = blob.data
150
ie.text_size = len(blob.data)
151
ie.text_sha1 = osutils.sha_string(blob.data)
152
# Check what revision we should store
154
for pinv in parent_invs:
155
if not file_id in pinv:
157
if pinv[file_id].text_sha1 == ie.text_sha1:
158
# found a revision in one of the parents to use
159
ie.revision = pinv[file_id].revision
161
parent_keys.append((file_id, pinv[file_id].revision))
162
if ie.revision is None:
163
# Need to store a new revision
164
ie.revision = revision_id
165
assert file_id is not None
166
assert ie.revision is not None
167
texts.add_lines((file_id, ie.revision), parent_keys,
168
osutils.split_lines(blob.data))
169
shagitmap.add_entry(hexsha, "blob", (ie.file_id, ie.revision))
170
if file_id in base_inv:
171
old_path = base_inv.id2path(file_id)
174
return [(old_path, path, file_id, ie)]
177
def import_git_tree(texts, mapping, path, hexsha, base_inv, parent_id,
178
revision_id, parent_invs, shagitmap, lookup_object):
92
def import_git_tree(repo, mapping, path, tree, inv, parent_invs, lookup_object):
179
93
"""Import a git tree object into a bzr repository.
181
:param texts: VersionedFiles object to add to
95
:param repo: A Bzr repository object
182
96
:param path: Path in the tree
183
97
:param tree: A git tree object
184
:param base_inv: Base inventory against which to return inventory delta
185
:return: Inventory delta for this subtree
98
:param inv: Inventory object
188
100
file_id = mapping.generate_file_id(path)
189
# We just have to hope this is indeed utf-8:
190
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
192
if not file_id in base_inv:
193
# Newly appeared here
194
ie.revision = revision_id
195
texts.add_lines((file_id, ie.revision), [], [])
196
ret.append((None, path, file_id, ie))
198
# See if this has changed at all
200
base_sha = shagitmap.lookup_tree(path, base_inv.revision_id)
204
if base_sha == hexsha:
205
# If nothing has changed since the base revision, we're done
207
# Remember for next time
208
existing_children = set()
209
shagitmap.add_entry(hexsha, "tree", (file_id, revision_id))
210
tree = lookup_object(hexsha)
101
text_revision = inv.revision_id
102
repo.texts.add_lines((file_id, text_revision),
103
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
105
ie = inv.add_path(path, "directory", file_id)
106
ie.revision = text_revision
211
107
for mode, name, hexsha in tree.entries():
212
108
entry_kind = (mode & 0700000) / 0100000
213
109
basename = name.decode("utf-8")
214
existing_children.add(basename)
216
111
child_path = name
218
113
child_path = urlutils.join(path, name)
219
114
if entry_kind == 0:
220
ret.extend(import_git_tree(texts, mapping, child_path, hexsha, base_inv,
221
file_id, revision_id, parent_invs, shagitmap, lookup_object))
115
tree = lookup_object(hexsha)
116
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, lookup_object)
222
117
elif entry_kind == 1:
118
blob = lookup_object(hexsha)
223
119
fs_mode = mode & 0777
224
file_kind = (mode & 070000) / 010000
225
if file_kind == 0: # regular file
230
raise AssertionError("Unknown file kind, mode=%r" % (mode,))
231
ret.extend(import_git_blob(texts, mapping, child_path, hexsha, base_inv,
232
file_id, revision_id, parent_invs, shagitmap, lookup_object,
233
bool(fs_mode & 0111), symlink))
120
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, bool(fs_mode & 0111))
235
raise AssertionError("Unknown object kind, perms=%r." % (mode,))
236
# Remove any children that have disappeared
237
if file_id in base_inv:
238
deletable = [v for k,v in base_inv[file_id].children.iteritems() if k not in existing_children]
241
ret.append((base_inv.id2path(ie.file_id), None, ie.file_id, None))
242
if ie.kind == "directory":
243
deletable.extend(ie.children.values())
247
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
122
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
125
def import_git_objects(repo, mapping, object_iter, pb=None):
249
126
"""Import a set of git objects into a bzr repository.
251
128
:param repo: Bazaar repository
253
130
:param object_iter: Iterator over Git objects.
255
132
# TODO: a more (memory-)efficient implementation of this
134
for i, o in enumerate(object_iter):
136
pb.update("fetching objects", i)
261
parent_invs_cache = LRUCache(50)
262
141
# Find and convert commit objects
265
pb.update("finding revisions to fetch", len(graph), None)
267
assert isinstance(head, str)
269
o = object_iter[head]
142
for o in objects.itervalues():
272
143
if isinstance(o, Commit):
273
144
rev = mapping.import_commit(o)
274
if repo.has_revision(rev.revision_id):
276
root_trees[rev.revision_id] = o.tree
145
root_trees[rev.revision_id] = objects[o.tree]
277
146
revisions[rev.revision_id] = rev
278
147
graph.append((rev.revision_id, rev.parent_ids))
279
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(),
280
"commit", (rev.revision_id, o._tree))
281
heads.extend([p for p in o.parents if p not in checked])
282
elif isinstance(o, Tag):
283
heads.append(o.object[1])
285
trace.warning("Unable to import head object %r" % o)
287
148
# Order the revisions
288
149
# Create the inventory objects
289
150
for i, revid in enumerate(topo_sort(graph)):
290
151
if pb is not None:
291
152
pb.update("fetching revisions", i, len(graph))
153
root_tree = root_trees[revid]
292
154
rev = revisions[revid]
293
155
# We have to do this here, since we have to walk the tree and
294
# we need to make sure to import the blobs / trees with the right
156
# we need to make sure to import the blobs / trees with the riht
295
157
# path; this may involve adding them more than once.
159
inv.revision_id = rev.revision_id
296
160
def lookup_object(sha):
298
return object_iter[sha]
300
return target_git_object_retriever[sha]
302
for parent_id in rev.parent_ids:
304
parent_invs.append(parent_invs_cache[parent_id])
306
parent_inv = repo.get_inventory(parent_id)
307
parent_invs.append(parent_inv)
308
parent_invs_cache[parent_id] = parent_inv
309
if parent_invs == []:
310
base_inv = Inventory(root_id=None)
312
base_inv = parent_invs[0]
313
inv_delta = import_git_tree(repo.texts, mapping, "",
314
root_trees[revid], base_inv, None, revid, parent_invs,
315
target_git_object_retriever._idmap, lookup_object)
317
basis_id = rev.parent_ids[0]
319
basis_id = NULL_REVISION
320
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
321
inv_delta, rev.revision_id, rev.parent_ids)
322
parent_invs_cache[rev.revision_id] = inv
323
repo.add_revision(rev.revision_id, rev)
324
target_git_object_retriever._idmap.commit()
163
return reconstruct_git_object(repo, mapping, sha)
164
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
165
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
167
repo.add_revision(rev.revision_id, rev, inv)
170
def reconstruct_git_commit(repo, rev):
171
raise NotImplementedError(self.reconstruct_git_commit)
174
def reconstruct_git_object(repo, mapping, sha):
176
revid = mapping.revision_id_foreign_to_bzr(sha)
178
rev = repo.get_revision(revid)
179
except NoSuchRevision:
182
return reconstruct_git_commit(rev)
186
raise KeyError("No such object %s" % sha)
327
189
class InterGitNonGitRepository(InterRepository):
328
"""Base InterRepository that copies revisions from a Git into a non-Git
331
_matching_repo_format = GitRepositoryFormat()
191
_matching_repo_format = GitFormat()
334
194
def _get_repo_format_to_test():
338
198
"""See InterRepository.copy_content."""
339
199
self.fetch(revision_id, pb, find_ghosts=False)
341
def fetch(self, revision_id=None, pb=None, find_ghosts=False, mapping=None,
343
self.fetch_refs(revision_id=revision_id, pb=pb, find_ghosts=find_ghosts,
344
mapping=mapping, fetch_spec=fetch_spec)
346
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
347
mapping=None, fetch_spec=None):
201
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
348
203
if mapping is None:
349
204
mapping = self.source.get_mapping()
350
if revision_id is not None:
351
interesting_heads = [revision_id]
352
elif fetch_spec is not None:
353
interesting_heads = fetch_spec.heads
355
interesting_heads = None
357
def determine_wants(refs):
359
if interesting_heads is None:
360
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
206
pb.note("git: %s", text)
207
def determine_wants(heads):
208
if revision_id is None:
362
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads]
211
ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
363
212
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
364
self.fetch_objects(determine_wants, mapping, pb)
369
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
370
"""InterRepository that copies revisions from a remote Git into a non-Git
373
def fetch_objects(self, determine_wants, mapping, pb=None):
375
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
376
213
graph_walker = BzrFetchGraphWalker(self.target, mapping)
379
216
create_pb = pb = ui.ui_factory.nested_progress_bar()
380
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
383
def record_determine_wants(heads):
384
wants = determine_wants(heads)
385
recorded_wants.extend(wants)
389
self.target.lock_write()
391
self.target.start_write_group()
393
objects_iter = self.source.fetch_objects(
394
record_determine_wants,
396
target_git_object_retriever.get_raw,
398
import_git_objects(self.target, mapping, objects_iter,
399
target_git_object_retriever, recorded_wants, pb)
401
self.target.commit_write_group()
409
def is_compatible(source, target):
410
"""Be compatible with GitRepository."""
411
# FIXME: Also check target uses VersionedFile
412
return (isinstance(source, RemoteGitRepository) and
413
target.supports_rich_root() and
414
not isinstance(target, GitRepository))
417
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
418
"""InterRepository that copies revisions from a remote Git into a non-Git
421
def fetch_objects(self, determine_wants, mapping, pb=None):
422
wants = determine_wants(self.source._git.get_refs())
425
create_pb = pb = ui.ui_factory.nested_progress_bar()
426
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
428
self.target.lock_write()
430
self.target.start_write_group()
432
import_git_objects(self.target, mapping,
433
self.source._git.object_store,
434
target_git_object_retriever, wants, pb)
436
self.target.commit_write_group()
444
def is_compatible(source, target):
445
"""Be compatible with GitRepository."""
446
# FIXME: Also check target uses VersionedFile
447
return (isinstance(source, LocalGitRepository) and
218
self.target.lock_write()
220
self.target.start_write_group()
222
import_git_objects(self.target, mapping,
223
iter(self.source.fetch_objects(determine_wants, graph_walker,
226
self.target.commit_write_group()
234
def is_compatible(source, target):
235
"""Be compatible with GitRepository."""
236
# FIXME: Also check target uses VersionedFile
237
return (isinstance(source, GitRepository) and
448
238
target.supports_rich_root() and
449
239
not isinstance(target, GitRepository))
452
242
class InterGitRepository(InterRepository):
453
"""InterRepository that copies between Git repositories."""
455
_matching_repo_format = GitRepositoryFormat()
244
_matching_repo_format = GitFormat()
458
247
def _get_repo_format_to_test():
added
removed
fetch.py
