To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.200.221
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.221
- Committer: Jelmer Vernooij
- Date: 2009-02-08 15:55:41 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20090208155541-nz99rcyajxi5fp8g
Add FOSDEM roundtripping notes.
- files added:
- foreign
- files removed:
- HACKING
- files renamed:
- tests/test_mapping.py => tests/test_ids.py
- files modified:
- Makefile
added
removed
fetch.py
1
# Copyright (C) 2008-2010 Jelmer Vernooij <jelmer@samba.org>
1
# Copyright (C) 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
from dulwich.objects import (
18
Commit,
19
Tag,
20
Tree,
21
S_ISGITLINK,
22
)
23
from dulwich.object_store import (
24
tree_lookup_path,
25
)
26
from itertools import (
27
imap,
28
)
29
import posixpath
30
import re
31
import stat
32
33
from bzrlib import (
34
debug,
35
osutils,
36
trace,
37
ui,
38
)
39
from bzrlib.errors import (
40
BzrError,
41
NoSuchId,
42
)
43
from bzrlib.inventory import (
44
Inventory,
45
InventoryDirectory,
46
InventoryFile,
47
InventoryLink,
48
TreeReference,
49
)
50
from bzrlib.repository import (
51
InterRepository,
52
)
53
from bzrlib.revision import (
54
NULL_REVISION,
55
)
56
from bzrlib.revisiontree import (
57
RevisionTree,
58
)
59
from bzrlib.testament import (
60
StrictTestament3,
61
)
62
from bzrlib.tsort import (
63
topo_sort,
64
)
65
from bzrlib.versionedfile import (
66
ChunkedContentFactory,
67
)
68
69
from bzrlib.plugins.git.mapping import (
70
DEFAULT_FILE_MODE,
71
mode_is_executable,
72
mode_kind,
73
warn_unusual_mode,
74
)
75
from bzrlib.plugins.git.object_store import (
76
BazaarObjectStore,
77
LRUTreeCache,
78
_tree_to_objects,
79
)
80
from bzrlib.plugins.git.remote import (
81
RemoteGitRepository,
82
)
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
23
83
24
from bzrlib.plugins.git.repository import (
84
GitRepository,
85
GitRepositoryFormat,
86
LocalGitRepository,
87
)
88
89
90
def import_git_blob(texts, mapping, path, name, (base_hexsha, hexsha),
91
base_inv, parent_id, revision_id,
92
parent_invs, lookup_object, (base_mode, mode), store_updater,
93
lookup_file_id):
25
LocalGitRepository,
26
GitRepository,
27
GitFormat,
28
)
29
from bzrlib.plugins.git.remote import RemoteGitRepository
30
31
import dulwich as git
32
from dulwich.client import SimpleFetchGraphWalker
33
from dulwich.objects import Commit
34
35
from cStringIO import StringIO
36
37
38
class BzrFetchGraphWalker(object):
39
"""GraphWalker implementation that uses a Bazaar repository."""
40
41
def __init__(self, repository, mapping):
42
self.repository = repository
43
self.mapping = mapping
44
self.done = set()
45
self.heads = set(repository.all_revision_ids())
46
self.parents = {}
47
48
def __iter__(self):
49
return iter(self.next, None)
50
51
def ack(self, sha):
52
revid = self.mapping.revision_id_foreign_to_bzr(sha)
53
self.remove(revid)
54
55
def remove(self, revid):
56
self.done.add(revid)
57
if revid in self.heads:
58
self.heads.remove(revid)
59
if revid in self.parents:
60
for p in self.parents[revid]:
61
self.remove(p)
62
63
def next(self):
64
while self.heads:
65
ret = self.heads.pop()
66
ps = self.repository.get_parent_map([ret])[ret]
67
self.parents[ret] = ps
68
self.heads.update([p for p in ps if not p in self.done])
69
try:
70
self.done.add(ret)
71
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
72
except InvalidRevisionId:
73
pass
74
return None
75
76
77
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, executable):
94
78
"""Import a git blob object into a bzr repository.
95
79
96
:param texts: VersionedFiles to add to
80
:param repo: bzr repository
97
81
:param path: Path in the tree
98
82
:param blob: A git blob
99
:return: Inventory delta for this file
100
"""
101
if mapping.is_control_file(path):
102
return []
103
if base_hexsha == hexsha and base_mode == mode:
104
# If nothing has changed since the base revision, we're done
105
return []
106
file_id = lookup_file_id(path)
107
if stat.S_ISLNK(mode):
108
cls = InventoryLink
109
else:
110
cls = InventoryFile
111
ie = cls(file_id, name.decode("utf-8"), parent_id)
112
if ie.kind == "file":
113
ie.executable = mode_is_executable(mode)
114
if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):
115
base_ie = base_inv[base_inv.path2id(path)]
116
ie.text_size = base_ie.text_size
117
ie.text_sha1 = base_ie.text_sha1
118
if ie.kind == "symlink":
119
ie.symlink_target = base_ie.symlink_target
120
if ie.executable == base_ie.executable:
121
ie.revision = base_ie.revision
122
else:
123
blob = lookup_object(hexsha)
124
else:
125
blob = lookup_object(hexsha)
126
if ie.kind == "symlink":
127
ie.revision = None
128
ie.symlink_target = blob.data
129
else:
130
ie.text_size = sum(imap(len, blob.chunked))
131
ie.text_sha1 = osutils.sha_strings(blob.chunked)
132
# Check what revision we should store
133
parent_keys = []
134
for pinv in parent_invs:
135
try:
136
pie = pinv[file_id]
137
except NoSuchId:
138
continue
139
if (pie.text_sha1 == ie.text_sha1 and
140
pie.executable == ie.executable and
141
pie.symlink_target == ie.symlink_target):
142
# found a revision in one of the parents to use
143
ie.revision = pie.revision
144
break
145
parent_key = (file_id, pie.revision)
146
if not parent_key in parent_keys:
147
parent_keys.append(parent_key)
148
if ie.revision is None:
149
# Need to store a new revision
150
ie.revision = revision_id
151
assert ie.revision is not None
152
if ie.kind == 'symlink':
153
chunks = []
154
else:
155
chunks = blob.chunked
156
texts.insert_record_stream([
157
ChunkedContentFactory((file_id, ie.revision),
158
tuple(parent_keys), ie.text_sha1, chunks)])
159
invdelta = []
160
if base_hexsha is not None:
161
old_path = path.decode("utf-8") # Renames are not supported yet
162
if stat.S_ISDIR(base_mode):
163
invdelta.extend(remove_disappeared_children(base_inv, old_path,
164
lookup_object(base_hexsha), [], lookup_object))
165
else:
166
old_path = None
167
new_path = path.decode("utf-8")
168
invdelta.append((old_path, new_path, file_id, ie))
169
if base_hexsha != hexsha:
170
store_updater.add_object(blob, ie, path)
171
return invdelta
172
173
174
class SubmodulesRequireSubtrees(BzrError):
175
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
176
internal = False
177
178
179
def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),
180
base_inv, parent_id, revision_id, parent_invs, lookup_object,
181
(base_mode, mode), store_updater, lookup_file_id):
182
if base_hexsha == hexsha and base_mode == mode:
183
return [], {}
184
file_id = lookup_file_id(path)
185
ie = TreeReference(file_id, name.decode("utf-8"), parent_id)
186
ie.revision = revision_id
187
if base_hexsha is None:
188
oldpath = None
189
else:
190
oldpath = path
191
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
192
texts.insert_record_stream([
193
ChunkedContentFactory((file_id, ie.revision), (), None, [])])
194
invdelta = [(oldpath, path, file_id, ie)]
195
return invdelta, {}
196
197
198
def remove_disappeared_children(base_inv, path, base_tree, existing_children,
199
lookup_object):
200
"""Generate an inventory delta for removed children.
201
202
:param base_inv: Base inventory against which to generate the
203
inventory delta.
204
:param path: Path to process (unicode)
205
:param base_tree: Git Tree base object
206
:param existing_children: Children that still exist
207
:param lookup_object: Lookup a git object by its SHA1
208
:return: Inventory delta, as list
209
"""
210
assert type(path) is unicode
211
ret = []
212
for name, mode, hexsha in base_tree.iteritems():
213
if name in existing_children:
214
continue
215
c_path = posixpath.join(path, name.decode("utf-8"))
216
file_id = base_inv.path2id(c_path)
217
assert file_id is not None
218
ret.append((c_path, None, file_id, None))
219
if stat.S_ISDIR(mode):
220
ret.extend(remove_disappeared_children(
221
base_inv, c_path, lookup_object(hexsha), [], lookup_object))
222
return ret
223
224
225
def import_git_tree(texts, mapping, path, name, (base_hexsha, hexsha),
226
base_inv, parent_id, revision_id, parent_invs,
227
lookup_object, (base_mode, mode), store_updater,
228
lookup_file_id, allow_submodules=False):
83
"""
84
file_id = mapping.generate_file_id(path)
85
text_revision = inv.revision_id
86
repo.texts.add_lines((file_id, text_revision),
87
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
88
osutils.split_lines(blob.data))
89
ie = inv.add_path(path, "file", file_id)
90
ie.revision = text_revision
91
ie.text_size = len(blob.data)
92
ie.text_sha1 = osutils.sha_string(blob.data)
93
ie.executable = executable
94
95
96
def import_git_tree(repo, mapping, path, tree, inv, parent_invs, lookup_object):
229
97
"""Import a git tree object into a bzr repository.
230
98
231
:param texts: VersionedFiles object to add to
232
:param path: Path in the tree (str)
233
:param name: Name of the tree (str)
99
:param repo: A Bzr repository object
100
:param path: Path in the tree
234
101
:param tree: A git tree object
235
:param base_inv: Base inventory against which to return inventory delta
236
:return: Inventory delta for this subtree
102
:param inv: Inventory object
237
103
"""
238
assert type(path) is str
239
assert type(name) is str
240
if base_hexsha == hexsha and base_mode == mode:
241
# If nothing has changed since the base revision, we're done
242
return [], {}
243
invdelta = []
244
file_id = lookup_file_id(path)
245
# We just have to hope this is indeed utf-8:
246
ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)
247
tree = lookup_object(hexsha)
248
if base_hexsha is None:
249
base_tree = None
250
old_path = None # Newly appeared here
251
else:
252
base_tree = lookup_object(base_hexsha)
253
old_path = path.decode("utf-8") # Renames aren't supported yet
254
new_path = path.decode("utf-8")
255
if base_tree is None or type(base_tree) is not Tree:
256
ie.revision = revision_id
257
invdelta.append((old_path, new_path, ie.file_id, ie))
258
texts.insert_record_stream([
259
ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
260
# Remember for next time
261
existing_children = set()
262
child_modes = {}
263
for child_mode, name, child_hexsha in tree.entries():
264
existing_children.add(name)
265
child_path = posixpath.join(path, name)
266
if type(base_tree) is Tree:
267
try:
268
child_base_mode, child_base_hexsha = base_tree[name]
269
except KeyError:
270
child_base_hexsha = None
271
child_base_mode = 0
272
else:
273
child_base_hexsha = None
274
child_base_mode = 0
275
if stat.S_ISDIR(child_mode):
276
subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
277
child_path, name, (child_base_hexsha, child_hexsha), base_inv,
278
file_id, revision_id, parent_invs, lookup_object,
279
(child_base_mode, child_mode), store_updater, lookup_file_id,
280
allow_submodules=allow_submodules)
281
elif S_ISGITLINK(child_mode): # submodule
282
if not allow_submodules:
283
raise SubmodulesRequireSubtrees()
284
subinvdelta, grandchildmodes = import_git_submodule(texts, mapping,
285
child_path, name, (child_base_hexsha, child_hexsha), base_inv,
286
file_id, revision_id, parent_invs, lookup_object,
287
(child_base_mode, child_mode), store_updater, lookup_file_id)
288
else:
289
subinvdelta = import_git_blob(texts, mapping, child_path, name,
290
(child_base_hexsha, child_hexsha), base_inv, file_id,
291
revision_id, parent_invs, lookup_object,
292
(child_base_mode, child_mode), store_updater, lookup_file_id)
293
grandchildmodes = {}
294
child_modes.update(grandchildmodes)
295
invdelta.extend(subinvdelta)
296
if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
297
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
298
child_modes[child_path] = child_mode
299
# Remove any children that have disappeared
300
if base_tree is not None and type(base_tree) is Tree:
301
invdelta.extend(remove_disappeared_children(base_inv, old_path,
302
base_tree, existing_children, lookup_object))
303
store_updater.add_object(tree, ie, path)
304
return invdelta, child_modes
305
306
307
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
308
o, rev, ret_tree, parent_trees, mapping, unusual_modes):
309
new_unusual_modes = mapping.export_unusual_file_modes(rev)
310
if new_unusual_modes != unusual_modes:
311
raise AssertionError("unusual modes don't match: %r != %r" % (
312
unusual_modes, new_unusual_modes))
313
# Verify that we can reconstruct the commit properly
314
rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)
315
if rec_o != o:
316
raise AssertionError("Reconstructed commit differs: %r != %r" % (
317
rec_o, o))
318
diff = []
319
new_objs = {}
320
for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,
321
target_git_object_retriever._cache.idmap, unusual_modes, mapping.BZR_DUMMY_FILE):
322
old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]
323
new_objs[path] = obj
324
if obj.id != old_obj_id:
325
diff.append((path, lookup_object(old_obj_id), obj))
326
for (path, old_obj, new_obj) in diff:
327
while (old_obj.type_name == "tree" and
328
new_obj.type_name == "tree" and
329
sorted(old_obj) == sorted(new_obj)):
330
for name in old_obj:
331
if old_obj[name][0] != new_obj[name][0]:
332
raise AssertionError("Modes for %s differ: %o != %o" %
333
(path, old_obj[name][0], new_obj[name][0]))
334
if old_obj[name][1] != new_obj[name][1]:
335
# Found a differing child, delve deeper
336
path = posixpath.join(path, name)
337
old_obj = lookup_object(old_obj[name][1])
338
new_obj = new_objs[path]
339
break
340
raise AssertionError("objects differ for %s: %r != %r" % (path,
341
old_obj, new_obj))
342
343
344
def import_git_commit(repo, mapping, head, lookup_object,
345
target_git_object_retriever, trees_cache):
346
o = lookup_object(head)
347
rev, roundtrip_revid, verifiers = mapping.import_commit(o,
348
lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])
349
# We have to do this here, since we have to walk the tree and
350
# we need to make sure to import the blobs / trees with the right
351
# path; this may involve adding them more than once.
352
parent_trees = trees_cache.revision_trees(rev.parent_ids)
353
if parent_trees == []:
354
base_inv = Inventory(root_id=None)
355
base_tree = None
356
base_mode = None
357
else:
358
base_inv = parent_trees[0].inventory
359
base_tree = lookup_object(o.parents[0]).tree
360
base_mode = stat.S_IFDIR
361
store_updater = target_git_object_retriever._get_updater(rev)
362
fileid_map = mapping.get_fileid_map(lookup_object, o.tree)
363
inv_delta, unusual_modes = import_git_tree(repo.texts,
364
mapping, "", "", (base_tree, o.tree), base_inv,
365
None, rev.revision_id, [p.inventory for p in parent_trees],
366
lookup_object, (base_mode, stat.S_IFDIR), store_updater,
367
fileid_map.lookup_file_id,
368
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
369
if unusual_modes != {}:
370
for path, mode in unusual_modes.iteritems():
371
warn_unusual_mode(rev.foreign_revid, path, mode)
372
mapping.import_unusual_file_modes(rev, unusual_modes)
373
try:
374
basis_id = rev.parent_ids[0]
375
except IndexError:
376
basis_id = NULL_REVISION
377
base_inv = None
378
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
379
inv_delta, rev.revision_id, rev.parent_ids, base_inv)
380
# FIXME: Check verifiers
381
testament = StrictTestament3(rev, inv)
382
calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
383
if roundtrip_revid is not None:
384
original_revid = rev.revision_id
385
rev.revision_id = roundtrip_revid
386
if calculated_verifiers != verifiers:
387
trace.mutter("Testament SHA1 %r for %r did not match %r.",
388
calculated_verifiers["testament3-sha1"],
389
rev.revision_id, verifiers["testament3-sha1"])
390
rev.revision_id = original_revid
391
store_updater.add_object(o, calculated_verifiers, None)
392
store_updater.finish()
393
ret_tree = RevisionTree(repo, inv, rev.revision_id)
394
trees_cache.add(ret_tree)
395
repo.add_revision(rev.revision_id, rev)
396
if "verify" in debug.debug_flags:
397
verify_commit_reconstruction(target_git_object_retriever,
398
lookup_object, o, rev, ret_tree, parent_trees, mapping,
399
unusual_modes)
400
401
402
def import_git_objects(repo, mapping, object_iter,
403
target_git_object_retriever, heads, pb=None, limit=None):
104
file_id = mapping.generate_file_id(path)
105
text_revision = inv.revision_id
106
repo.texts.add_lines((file_id, text_revision),
107
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
108
[])
109
ie = inv.add_path(path, "directory", file_id)
110
ie.revision = text_revision
111
for mode, name, hexsha in tree.entries():
112
entry_kind = (mode & 0700000) / 0100000
113
basename = name.decode("utf-8")
114
if path == "":
115
child_path = name
116
else:
117
child_path = urlutils.join(path, name)
118
if entry_kind == 0:
119
tree = lookup_object(hexsha)
120
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, lookup_object)
121
elif entry_kind == 1:
122
blob = lookup_object(hexsha)
123
fs_mode = mode & 0777
124
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, bool(fs_mode & 0111))
125
else:
126
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
127
128
129
def import_git_objects(repo, mapping, object_iter, pb=None):
404
130
"""Import a set of git objects into a bzr repository.
405
131
406
:param repo: Target Bazaar repository
132
:param repo: Bazaar repository
407
133
:param mapping: Mapping to use
408
134
:param object_iter: Iterator over Git objects.
409
:return: Tuple with pack hints and last imported revision id
410
135
"""
411
def lookup_object(sha):
412
try:
413
return object_iter[sha]
414
except KeyError:
415
return target_git_object_retriever[sha]
136
# TODO: a more (memory-)efficient implementation of this
416
137
graph = []
417
checked = set()
418
heads = list(set(heads))
419
trees_cache = LRUTreeCache(repo)
138
root_trees = {}
139
revisions = {}
420
140
# Find and convert commit objects
421
while heads:
422
if pb is not None:
423
pb.update("finding revisions to fetch", len(graph), None)
424
head = heads.pop()
425
assert isinstance(head, str)
426
try:
427
o = lookup_object(head)
428
except KeyError:
429
continue
141
for o in object_iter.iterobjects():
430
142
if isinstance(o, Commit):
431
rev, roundtrip_revid, verifiers = mapping.import_commit(o,
432
lambda x: None)
433
if (repo.has_revision(rev.revision_id) or
434
(roundtrip_revid and repo.has_revision(roundtrip_revid))):
435
continue
436
graph.append((o.id, o.parents))
437
heads.extend([p for p in o.parents if p not in checked])
438
elif isinstance(o, Tag):
439
if o.object[1] not in checked:
440
heads.append(o.object[1])
441
else:
442
trace.warning("Unable to import head object %r" % o)
443
checked.add(o.id)
444
del checked
143
rev = mapping.import_commit(o)
144
root_trees[rev.revision_id] = object_iter[o.tree]
145
revisions[rev.revision_id] = rev
146
graph.append((rev.revision_id, rev.parent_ids))
445
147
# Order the revisions
446
148
# Create the inventory objects
447
batch_size = 1000
448
revision_ids = topo_sort(graph)
449
pack_hints = []
450
if limit is not None:
451
revision_ids = revision_ids[:limit]
452
last_imported = None
453
for offset in range(0, len(revision_ids), batch_size):
454
target_git_object_retriever.start_write_group()
455
try:
456
repo.start_write_group()
457
try:
458
for i, head in enumerate(
459
revision_ids[offset:offset+batch_size]):
460
if pb is not None:
461
pb.update("fetching revisions", offset+i,
462
len(revision_ids))
463
import_git_commit(repo, mapping, head, lookup_object,
464
target_git_object_retriever, trees_cache)
465
last_imported = head
466
except:
467
repo.abort_write_group()
468
raise
469
else:
470
hint = repo.commit_write_group()
471
if hint is not None:
472
pack_hints.extend(hint)
473
except:
474
target_git_object_retriever.abort_write_group()
475
raise
476
else:
477
target_git_object_retriever.commit_write_group()
478
return pack_hints, last_imported
479
480
481
class InterGitRepository(InterRepository):
482
483
_matching_repo_format = GitRepositoryFormat()
149
for i, revid in enumerate(topo_sort(graph)):
150
if pb is not None:
151
pb.update("fetching revisions", i, len(graph))
152
root_tree = root_trees[revid]
153
rev = revisions[revid]
154
# We have to do this here, since we have to walk the tree and
155
# we need to make sure to import the blobs / trees with the riht
156
# path; this may involve adding them more than once.
157
inv = Inventory()
158
inv.revision_id = rev.revision_id
159
def lookup_object(sha):
160
if sha in object_iter:
161
return object_iter[sha]
162
return reconstruct_git_object(repo, mapping, sha)
163
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
164
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
165
lookup_object)
166
repo.add_revision(rev.revision_id, rev, inv)
167
168
169
def reconstruct_git_commit(repo, rev):
170
raise NotImplementedError(self.reconstruct_git_commit)
171
172
173
def reconstruct_git_object(repo, mapping, sha):
174
# Commit
175
revid = mapping.revision_id_foreign_to_bzr(sha)
176
try:
177
rev = repo.get_revision(revid)
178
except NoSuchRevision:
179
pass
180
else:
181
return reconstruct_git_commit(rev)
182
183
# TODO: Tree
184
# TODO: Blob
185
raise KeyError("No such object %s" % sha)
186
187
188
class InterGitNonGitRepository(InterRepository):
189
190
_matching_repo_format = GitFormat()
484
191
485
192
@staticmethod
486
193
def _get_repo_format_to_test():
490
197
"""See InterRepository.copy_content."""
491
198
self.fetch(revision_id, pb, find_ghosts=False)
492
199
493
494
class InterGitNonGitRepository(InterGitRepository):
495
"""Base InterRepository that copies revisions from a Git into a non-Git
496
repository."""
497
498
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
499
"""Fetch objects from a remote server.
500
501
:param determine_wants: determine_wants callback
502
:param mapping: BzrGitMapping to use
503
:param pb: Optional progress bar
504
:param limit: Maximum number of commits to import.
505
:return: Tuple with pack hint, last imported revision id and remote refs
506
"""
507
raise NotImplementedError(self.fetch_objects)
508
509
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
510
mapping=None, fetch_spec=None):
200
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
201
mapping=None):
511
202
if mapping is None:
512
203
mapping = self.source.get_mapping()
513
if revision_id is not None:
514
interesting_heads = [revision_id]
515
elif fetch_spec is not None:
516
interesting_heads = fetch_spec.heads
517
else:
518
interesting_heads = None
519
def determine_wants(refs):
520
if interesting_heads is None:
521
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
522
else:
523
ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
524
return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]
525
(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)
526
if pack_hint is not None and self.target._format.pack_compresses:
527
self.target.pack(hint=pack_hint)
528
return remote_refs
529
530
531
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
532
def report_git_progress(pb, text):
533
text = text.rstrip("\r\n")
534
g = _GIT_PROGRESS_RE.match(text)
535
if g is not None:
536
(text, pct, current, total) = g.groups()
537
pb.update(text, int(current), int(total))
538
else:
539
pb.update(text, 0, 0)
540
541
542
class DetermineWantsRecorder(object):
543
544
def __init__(self, actual):
545
self.actual = actual
546
self.wants = []
547
self.remote_refs = {}
548
549
def __call__(self, refs):
550
self.remote_refs = refs
551
self.wants = self.actual(refs)
552
return self.wants
553
554
555
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
556
"""InterRepository that copies revisions from a remote Git into a non-Git
557
repository."""
558
559
def get_target_heads(self):
560
# FIXME: This should be more efficient
561
all_revs = self.target.all_revision_ids()
562
parent_map = self.target.get_parent_map(all_revs)
563
all_parents = set()
564
map(all_parents.update, parent_map.itervalues())
565
return set(all_revs) - all_parents
566
567
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
568
"""See `InterGitNonGitRepository`."""
569
204
def progress(text):
570
report_git_progress(pb, text)
571
store = BazaarObjectStore(self.target, mapping)
572
self.target.lock_write()
573
try:
574
heads = self.get_target_heads()
575
graph_walker = store.get_graph_walker(
576
[store._lookup_revision_sha1(head) for head in heads])
577
wants_recorder = DetermineWantsRecorder(determine_wants)
578
579
create_pb = None
580
if pb is None:
581
create_pb = pb = ui.ui_factory.nested_progress_bar()
582
try:
583
objects_iter = self.source.fetch_objects(
584
wants_recorder, graph_walker, store.get_raw,
585
progress)
586
(pack_hint, last_rev) = import_git_objects(self.target, mapping,
587
objects_iter, store, wants_recorder.wants, pb, limit)
588
return (pack_hint, last_rev, wants_recorder.remote_refs)
589
finally:
590
if create_pb:
591
create_pb.finished()
592
finally:
593
self.target.unlock()
594
595
@staticmethod
596
def is_compatible(source, target):
597
"""Be compatible with GitRepository."""
598
return (isinstance(source, RemoteGitRepository) and
599
target.supports_rich_root() and
600
not isinstance(target, GitRepository) and
601
target.texts is not None)
602
603
604
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
605
"""InterRepository that copies revisions from a local Git into a non-Git
606
repository."""
607
608
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
609
"""See `InterGitNonGitRepository`."""
610
remote_refs = self.source._git.get_refs()
611
wants = determine_wants(remote_refs)
205
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
206
def determine_wants(heads):
207
if revision_id is None:
208
ret = heads.values()
209
else:
210
ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
211
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
212
graph_walker = BzrFetchGraphWalker(self.target, mapping)
612
213
create_pb = None
613
214
if pb is None:
614
215
create_pb = pb = ui.ui_factory.nested_progress_bar()
615
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
616
216
try:
617
217
self.target.lock_write()
618
218
try:
619
(pack_hint, last_rev) = import_git_objects(self.target, mapping,
620
self.source._git.object_store,
621
target_git_object_retriever, wants, pb, limit)
622
return (pack_hint, last_rev, remote_refs)
219
self.target.start_write_group()
220
try:
221
objects_iter = self.source.fetch_objects(determine_wants,
222
graph_walker, progress)
223
import_git_objects(self.target, mapping, objects_iter, pb)
224
finally:
225
self.target.commit_write_group()
623
226
finally:
624
227
self.target.unlock()
625
228
finally:
629
232
@staticmethod
630
233
def is_compatible(source, target):
631
234
"""Be compatible with GitRepository."""
632
return (isinstance(source, LocalGitRepository) and
235
# FIXME: Also check target uses VersionedFile
236
return (isinstance(source, GitRepository) and
633
237
target.supports_rich_root() and
634
not isinstance(target, GitRepository) and
635
target.texts is not None)
636
637
638
class InterGitGitRepository(InterGitRepository):
639
"""InterRepository that copies between Git repositories."""
640
641
def fetch_objects(self, determine_wants, mapping, pb=None):
642
def progress(text):
643
trace.note("git: %s", text)
644
graphwalker = self.target._git.get_graph_walker()
645
if (isinstance(self.source, LocalGitRepository) and
646
isinstance(self.target, LocalGitRepository)):
647
refs = self.source._git.fetch(self.target._git, determine_wants,
648
progress)
649
return (None, None, refs)
650
elif (isinstance(self.source, LocalGitRepository) and
651
isinstance(self.target, RemoteGitRepository)):
652
raise NotImplementedError
653
elif (isinstance(self.source, RemoteGitRepository) and
654
isinstance(self.target, LocalGitRepository)):
655
f, commit = self.target._git.object_store.add_thin_pack()
656
try:
657
refs = self.source.bzrdir.root_transport.fetch_pack(
658
determine_wants, graphwalker, f.write, progress)
659
commit()
660
return (None, None, refs)
661
except:
662
f.close()
663
raise
664
else:
665
raise AssertionError
666
667
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
668
mapping=None, fetch_spec=None, branches=None):
238
not isinstance(target, GitRepository))
239
240
241
class InterGitRepository(InterRepository):
242
243
_matching_repo_format = GitFormat()
244
245
@staticmethod
246
def _get_repo_format_to_test():
247
return None
248
249
def copy_content(self, revision_id=None, pb=None):
250
"""See InterRepository.copy_content."""
251
self.fetch(revision_id, pb, find_ghosts=False)
252
253
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
254
mapping=None):
669
255
if mapping is None:
670
256
mapping = self.source.get_mapping()
257
def progress(text):
258
info("git: %s", text)
671
259
r = self.target._git
672
if revision_id is not None:
260
if revision_id is None:
261
determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
262
else:
673
263
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
674
elif fetch_spec is not None:
675
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
676
if branches is not None:
677
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
678
elif fetch_spec is None and revision_id is None:
679
determine_wants = r.object_store.determine_wants_all
680
else:
681
264
determine_wants = lambda x: [y for y in args if not y in r.object_store]
682
self.fetch_objects(determine_wants, mapping)
265
266
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
267
f, commit = r.object_store.add_pack()
268
try:
269
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
270
f.close()
271
commit()
272
except:
273
f.close()
274
raise
683
275
684
276
@staticmethod
685
277
def is_compatible(source, target):
686
278
"""Be compatible with GitRepository."""
687
return (isinstance(source, GitRepository) and
279
return (isinstance(source, GitRepository) and
688
280
isinstance(target, GitRepository))
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0