To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.200.231
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.231
- Committer: Jelmer Vernooij
- Date: 2009-02-12 02:31:01 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20090212023101-s7vcsvibldg5enz2
Partially fix pull.
- files added:
- foreign
- files removed:
- HACKING
- files renamed:
- object_store.py => converter.py
- tests/test_mapping.py => tests/test_ids.py
- files modified:
- Makefile
added
removed
fetch.py
1
# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
1
# Copyright (C) 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
from dulwich.objects import (
18
Commit,
19
Tag,
20
Tree,
21
S_ISGITLINK,
22
)
23
from dulwich.object_store import (
24
tree_lookup_path,
25
)
26
import posixpath
27
import re
28
import stat
29
30
from bzrlib import (
31
debug,
32
osutils,
33
trace,
34
ui,
35
)
36
from bzrlib.errors import (
37
BzrError,
38
NoSuchId,
39
)
40
from bzrlib.inventory import (
41
Inventory,
42
InventoryDirectory,
43
InventoryFile,
44
InventoryLink,
45
TreeReference,
46
)
47
from bzrlib.repository import (
48
InterRepository,
49
)
50
from bzrlib.revision import (
51
NULL_REVISION,
52
)
53
from bzrlib.tsort import (
54
topo_sort,
55
)
56
from bzrlib.versionedfile import (
57
ChunkedContentFactory,
58
)
59
60
from bzrlib.plugins.git.mapping import (
61
DEFAULT_FILE_MODE,
62
inventory_to_tree_and_blobs,
63
mode_is_executable,
64
mode_kind,
65
squash_revision,
66
warn_unusual_mode,
67
)
68
from bzrlib.plugins.git.object_store import (
69
BazaarObjectStore,
70
LRUInventoryCache,
71
)
72
from bzrlib.plugins.git.remote import (
73
RemoteGitRepository,
74
)
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
23
75
24
from bzrlib.plugins.git.repository import (
76
GitRepository,
77
GitRepositoryFormat,
78
LocalGitRepository,
79
)
80
81
82
def import_git_blob(texts, mapping, path, name, (base_hexsha, hexsha),
83
base_inv, base_inv_shamap, parent_id, revision_id,
84
parent_invs, lookup_object, (base_mode, mode)):
25
LocalGitRepository,
26
GitRepository,
27
GitFormat,
28
)
29
from bzrlib.plugins.git.converter import GitObjectConverter
30
from bzrlib.plugins.git.remote import RemoteGitRepository
31
32
import dulwich as git
33
from dulwich.client import SimpleFetchGraphWalker
34
from dulwich.objects import Commit
35
36
from cStringIO import StringIO
37
38
39
class BzrFetchGraphWalker(object):
40
"""GraphWalker implementation that uses a Bazaar repository."""
41
42
def __init__(self, repository, mapping):
43
self.repository = repository
44
self.mapping = mapping
45
self.done = set()
46
self.heads = set(repository.all_revision_ids())
47
self.parents = {}
48
49
def __iter__(self):
50
return iter(self.next, None)
51
52
def ack(self, sha):
53
revid = self.mapping.revision_id_foreign_to_bzr(sha)
54
self.remove(revid)
55
56
def remove(self, revid):
57
self.done.add(revid)
58
if revid in self.heads:
59
self.heads.remove(revid)
60
if revid in self.parents:
61
for p in self.parents[revid]:
62
self.remove(p)
63
64
def next(self):
65
while self.heads:
66
ret = self.heads.pop()
67
ps = self.repository.get_parent_map([ret])[ret]
68
self.parents[ret] = ps
69
self.heads.update([p for p in ps if not p in self.done])
70
try:
71
self.done.add(ret)
72
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
73
except InvalidRevisionId:
74
pass
75
return None
76
77
78
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, gitmap, executable):
85
79
"""Import a git blob object into a bzr repository.
86
80
87
:param texts: VersionedFiles to add to
81
:param repo: bzr repository
88
82
:param path: Path in the tree
89
83
:param blob: A git blob
90
:return: Inventory delta for this file
91
84
"""
92
if base_hexsha == hexsha and base_mode == mode:
93
# If nothing has changed since the base revision, we're done
94
return [], []
95
file_id = mapping.generate_file_id(path)
96
if stat.S_ISLNK(mode):
97
cls = InventoryLink
98
else:
99
cls = InventoryFile
100
ie = cls(file_id, name.decode("utf-8"), parent_id)
101
ie.executable = mode_is_executable(mode)
102
if base_hexsha == hexsha and mode_kind(base_mode) == mode_kind(mode):
103
base_ie = base_inv[base_inv.path2id(path)]
104
ie.text_size = base_ie.text_size
105
ie.text_sha1 = base_ie.text_sha1
106
ie.symlink_target = base_ie.symlink_target
107
if ie.executable == base_ie.executable:
108
ie.revision = base_ie.revision
109
else:
110
blob = lookup_object(hexsha)
111
else:
112
blob = lookup_object(hexsha)
113
if ie.kind == "symlink":
114
ie.revision = None
115
ie.symlink_target = blob.data
116
ie.text_size = None
117
ie.text_sha1 = None
118
else:
119
ie.text_size = len(blob.data)
120
ie.text_sha1 = osutils.sha_string(blob.data)
121
# Check what revision we should store
122
parent_keys = []
123
for pinv in parent_invs[1:]:
124
if pinv.revision_id == base_inv.revision_id:
125
pie = base_ie
126
if pie is None:
127
continue
128
else:
129
try:
130
pie = pinv[file_id]
131
except NoSuchId:
132
continue
133
if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:
134
# found a revision in one of the parents to use
135
ie.revision = pie.revision
136
break
137
parent_keys.append((file_id, pie.revision))
138
if ie.revision is None:
139
# Need to store a new revision
140
ie.revision = revision_id
141
assert ie.revision is not None
142
if ie.kind == 'symlink':
143
chunks = []
144
else:
145
try:
146
chunks = blob.chunked
147
except AttributeError: # older version of dulwich
148
chunks = [blob.data]
149
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, chunks)])
150
invdelta = []
151
if base_hexsha is not None:
152
old_path = path # Renames are not supported yet
153
if stat.S_ISDIR(base_mode):
154
invdelta.extend(remove_disappeared_children(base_inv, old_path,
155
lookup_object(base_hexsha), [], lookup_object))
156
else:
157
old_path = None
158
invdelta.append((old_path, path, file_id, ie))
159
return (invdelta, [(ie.file_id, "blob", hexsha, ie.revision)])
160
161
162
class SubmodulesRequireSubtrees(BzrError):
163
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
164
internal = False
165
166
167
def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),
168
base_inv, parent_id, revision_id, parent_invs, lookup_object,
169
(base_mode, mode)):
170
if base_hexsha == hexsha and base_mode == mode:
171
return [], {}, {}
172
file_id = mapping.generate_file_id(path)
173
ie = TreeReference(file_id, name.decode("utf-8"), parent_id)
174
ie.revision = revision_id
175
if base_hexsha is None:
176
oldpath = None
177
else:
178
oldpath = path
179
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
180
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])
181
invdelta = [(oldpath, path, file_id, ie)]
182
return invdelta, {}, []
183
184
185
def remove_disappeared_children(base_inv, path, base_tree, existing_children,
186
lookup_object):
187
ret = []
188
for name, mode, hexsha in base_tree.iteritems():
189
if name in existing_children:
190
continue
191
c_path = posixpath.join(path, name.decode("utf-8"))
192
ret.append((c_path, None, base_inv.path2id(c_path), None))
193
if stat.S_ISDIR(mode):
194
ret.extend(remove_disappeared_children(
195
base_inv, c_path, lookup_object(hexsha), [], lookup_object))
196
return ret
197
198
199
def import_git_tree(texts, mapping, path, name, (base_hexsha, hexsha),
200
base_inv, base_inv_shamap, parent_id, revision_id, parent_invs,
201
lookup_object, (base_mode, mode), allow_submodules=False):
85
file_id = mapping.generate_file_id(path)
86
text_revision = inv.revision_id
87
repo.texts.add_lines((file_id, text_revision),
88
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
89
osutils.split_lines(blob.data))
90
ie = inv.add_path(path, "file", file_id)
91
ie.revision = text_revision
92
ie.text_size = len(blob.data)
93
ie.text_sha1 = osutils.sha_string(blob.data)
94
ie.executable = executable
95
gitmap._idmap.add_entry(blob.sha().hexdigest(), "blob", (ie.file_id, ie.revision))
96
97
98
def import_git_tree(repo, mapping, path, tree, inv, parent_invs,
99
gitmap, lookup_object):
202
100
"""Import a git tree object into a bzr repository.
203
101
204
:param texts: VersionedFiles object to add to
102
:param repo: A Bzr repository object
205
103
:param path: Path in the tree
206
104
:param tree: A git tree object
207
:param base_inv: Base inventory against which to return inventory delta
208
:return: Inventory delta for this subtree
105
:param inv: Inventory object
209
106
"""
210
if base_hexsha == hexsha and base_mode == mode:
211
# If nothing has changed since the base revision, we're done
212
return [], {}, []
213
invdelta = []
214
107
file_id = mapping.generate_file_id(path)
215
# We just have to hope this is indeed utf-8:
216
ie = InventoryDirectory(file_id, name.decode("utf-8"), parent_id)
217
tree = lookup_object(hexsha)
218
if base_hexsha is None:
219
base_tree = None
220
old_path = None # Newly appeared here
221
else:
222
base_tree = lookup_object(base_hexsha)
223
old_path = path # Renames aren't supported yet
224
if base_tree is None or type(base_tree) is not Tree:
225
ie.revision = revision_id
226
invdelta.append((old_path, path, ie.file_id, ie))
227
texts.insert_record_stream([ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
228
# Remember for next time
229
existing_children = set()
230
child_modes = {}
231
shamap = [(ie.file_id, "tree", hexsha, revision_id)]
232
for child_mode, name, child_hexsha in tree.entries():
233
existing_children.add(name)
234
child_path = posixpath.join(path, name)
235
if type(base_tree) is Tree:
236
try:
237
child_base_mode, child_base_hexsha = base_tree[name]
238
except KeyError:
239
child_base_hexsha = None
240
child_base_mode = 0
241
else:
242
child_base_hexsha = None
243
child_base_mode = 0
244
if stat.S_ISDIR(child_mode):
245
subinvdelta, grandchildmodes, subshamap = import_git_tree(
246
texts, mapping, child_path, name,
247
(child_base_hexsha, child_hexsha),
248
base_inv, base_inv_shamap,
249
file_id, revision_id, parent_invs, lookup_object,
250
(child_base_mode, child_mode),
251
allow_submodules=allow_submodules)
252
elif S_ISGITLINK(child_mode): # submodule
253
if not allow_submodules:
254
raise SubmodulesRequireSubtrees()
255
subinvdelta, grandchildmodes, subshamap = import_git_submodule(
256
texts, mapping, child_path, name,
257
(child_base_hexsha, child_hexsha),
258
base_inv, file_id, revision_id, parent_invs, lookup_object,
259
(child_base_mode, child_mode))
260
else:
261
subinvdelta, subshamap = import_git_blob(texts, mapping,
262
child_path, name, (child_base_hexsha, child_hexsha),
263
base_inv, base_inv_shamap,
264
file_id,
265
revision_id, parent_invs, lookup_object,
266
(child_base_mode, child_mode))
267
grandchildmodes = {}
268
child_modes.update(grandchildmodes)
269
invdelta.extend(subinvdelta)
270
shamap.extend(subshamap)
271
if child_mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
272
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
273
child_modes[child_path] = child_mode
274
# Remove any children that have disappeared
275
if base_tree is not None and type(base_tree) is Tree:
276
invdelta.extend(remove_disappeared_children(base_inv, old_path,
277
base_tree, existing_children, lookup_object))
278
return invdelta, child_modes, shamap
279
280
281
def import_git_commit(repo, mapping, head, lookup_object,
282
target_git_object_retriever, parent_invs_cache):
283
o = lookup_object(head)
284
rev = mapping.import_commit(o)
285
# We have to do this here, since we have to walk the tree and
286
# we need to make sure to import the blobs / trees with the right
287
# path; this may involve adding them more than once.
288
parent_invs = parent_invs_cache.get_inventories(rev.parent_ids)
289
if parent_invs == []:
290
base_inv = Inventory(root_id=None)
291
base_inv_shamap = None # Should never be accessed
292
base_tree = None
293
base_mode = None
294
else:
295
base_inv = parent_invs[0]
296
base_inv_shamap = target_git_object_retriever._idmap.get_inventory_sha_map(base_inv.revision_id)
297
base_tree = lookup_object(o.parents[0]).tree
298
base_mode = stat.S_IFDIR
299
inv_delta, unusual_modes, entries = import_git_tree(repo.texts,
300
mapping, "", u"", (base_tree, o.tree), base_inv, base_inv_shamap,
301
None, rev.revision_id, parent_invs, lookup_object,
302
(base_mode, stat.S_IFDIR),
303
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
304
target_git_object_retriever._idmap.add_entries(rev.revision_id,
305
rev.parent_ids, head, o.tree, entries)
306
if unusual_modes != {}:
307
for path, mode in unusual_modes.iteritems():
308
warn_unusual_mode(rev.foreign_revid, path, mode)
309
mapping.import_unusual_file_modes(rev, unusual_modes)
310
try:
311
basis_id = rev.parent_ids[0]
312
except IndexError:
313
basis_id = NULL_REVISION
314
base_inv = None
315
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
316
inv_delta, rev.revision_id, rev.parent_ids,
317
base_inv)
318
parent_invs_cache.add(rev.revision_id, inv)
319
repo.add_revision(rev.revision_id, rev)
320
if "verify" in debug.debug_flags:
321
new_unusual_modes = mapping.export_unusual_file_modes(rev)
322
if new_unusual_modes != unusual_modes:
323
raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))
324
objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)
325
for newsha1, newobj, path in objs:
326
assert path is not None
327
if path == "":
328
oldsha1 = o.tree
329
else:
330
(oldmode, oldsha1) = tree_lookup_path(lookup_object, o.tree, path)
331
if oldsha1 != newsha1:
332
raise AssertionError("%r != %r in %s" % (oldsha1, newsha1, path))
333
334
335
def import_git_objects(repo, mapping, object_iter,
336
target_git_object_retriever, heads, pb=None, limit=None):
108
text_revision = inv.revision_id
109
repo.texts.add_lines((file_id, text_revision),
110
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
111
[])
112
ie = inv.add_path(path, "directory", file_id)
113
ie.revision = text_revision
114
gitmap._idmap.add_entry(tree.sha().hexdigest(), "tree", (file_id, text_revision))
115
for mode, name, hexsha in tree.entries():
116
entry_kind = (mode & 0700000) / 0100000
117
basename = name.decode("utf-8")
118
if path == "":
119
child_path = name
120
else:
121
child_path = urlutils.join(path, name)
122
if entry_kind == 0:
123
tree = lookup_object(hexsha)
124
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, gitmap, lookup_object)
125
elif entry_kind == 1:
126
blob = lookup_object(hexsha)
127
fs_mode = mode & 0777
128
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, gitmap, bool(fs_mode & 0111))
129
else:
130
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
131
132
133
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
134
pb=None):
337
135
"""Import a set of git objects into a bzr repository.
338
136
339
:param repo: Target Bazaar repository
137
:param repo: Bazaar repository
340
138
:param mapping: Mapping to use
341
139
:param object_iter: Iterator over Git objects.
342
:return: Tuple with pack hints and last imported revision id
343
140
"""
344
def lookup_object(sha):
345
try:
346
return object_iter[sha]
347
except KeyError:
348
return target_git_object_retriever[sha]
141
# TODO: a more (memory-)efficient implementation of this
349
142
graph = []
350
checked = set()
351
heads = list(set(heads))
352
parent_invs_cache = LRUInventoryCache(repo)
143
root_trees = {}
144
revisions = {}
353
145
# Find and convert commit objects
354
while heads:
355
if pb is not None:
356
pb.update("finding revisions to fetch", len(graph), None)
357
head = heads.pop()
358
assert isinstance(head, str)
359
try:
360
o = lookup_object(head)
361
except KeyError:
362
continue
146
for o in object_iter.iterobjects():
363
147
if isinstance(o, Commit):
364
148
rev = mapping.import_commit(o)
365
if repo.has_revision(rev.revision_id):
366
continue
367
squash_revision(repo, rev)
368
graph.append((o.id, o.parents))
369
heads.extend([p for p in o.parents if p not in checked])
370
elif isinstance(o, Tag):
371
if o.object[1] not in checked:
372
heads.append(o.object[1])
373
else:
374
trace.warning("Unable to import head object %r" % o)
375
checked.add(o.id)
376
del checked
149
root_trees[rev.revision_id] = object_iter[o.tree]
150
revisions[rev.revision_id] = rev
151
graph.append((rev.revision_id, rev.parent_ids))
152
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(), "commit", (rev.revision_id, o._tree))
377
153
# Order the revisions
378
154
# Create the inventory objects
379
batch_size = 1000
380
revision_ids = topo_sort(graph)
381
pack_hints = []
382
if limit is not None:
383
revision_ids = revision_ids[:limit]
384
last_imported = None
385
for offset in range(0, len(revision_ids), batch_size):
386
target_git_object_retriever.start_write_group() # FIXME: try/finally
387
try:
388
repo.start_write_group()
389
try:
390
for i, head in enumerate(
391
revision_ids[offset:offset+batch_size]):
392
if pb is not None:
393
pb.update("fetching revisions", offset+i,
394
len(revision_ids))
395
import_git_commit(repo, mapping, head, lookup_object,
396
target_git_object_retriever,
397
parent_invs_cache)
398
last_imported = head
399
except:
400
repo.abort_write_group()
401
raise
402
else:
403
hint = repo.commit_write_group()
404
if hint is not None:
405
pack_hints.extend(hint)
406
except:
407
target_git_object_retriever.abort_write_group()
408
raise
409
else:
410
target_git_object_retriever.commit_write_group()
411
return pack_hints, last_imported
412
413
414
class InterGitRepository(InterRepository):
415
416
_matching_repo_format = GitRepositoryFormat()
155
for i, revid in enumerate(topo_sort(graph)):
156
if pb is not None:
157
pb.update("fetching revisions", i, len(graph))
158
root_tree = root_trees[revid]
159
rev = revisions[revid]
160
# We have to do this here, since we have to walk the tree and
161
# we need to make sure to import the blobs / trees with the riht
162
# path; this may involve adding them more than once.
163
inv = Inventory()
164
inv.revision_id = rev.revision_id
165
def lookup_object(sha):
166
if sha in object_iter:
167
return object_iter[sha]
168
return target_git_object_retriever[sha]
169
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
170
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
171
target_git_object_retriever, lookup_object)
172
repo.add_revision(rev.revision_id, rev, inv)
173
174
175
class InterGitNonGitRepository(InterRepository):
176
177
_matching_repo_format = GitFormat()
417
178
418
179
@staticmethod
419
180
def _get_repo_format_to_test():
423
184
"""See InterRepository.copy_content."""
424
185
self.fetch(revision_id, pb, find_ghosts=False)
425
186
426
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
427
mapping=None, fetch_spec=None):
428
self.fetch_refs(revision_id=revision_id, pb=pb,
429
find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)
430
431
432
class InterGitNonGitRepository(InterGitRepository):
433
"""Base InterRepository that copies revisions from a Git into a non-Git
434
repository."""
435
436
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
437
mapping=None, fetch_spec=None):
438
if mapping is None:
439
mapping = self.source.get_mapping()
440
if revision_id is not None:
441
interesting_heads = [revision_id]
442
elif fetch_spec is not None:
443
interesting_heads = fetch_spec.heads
444
else:
445
interesting_heads = None
446
self._refs = {}
447
def determine_wants(refs):
448
self._refs = refs
449
if interesting_heads is None:
450
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
451
else:
452
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
453
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
454
(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)
455
if pack_hint is not None and self.target._format.pack_compresses:
456
self.target.pack(hint=pack_hint)
457
if interesting_heads is not None:
458
present_interesting_heads = self.target.has_revisions(interesting_heads)
459
missing_interesting_heads = set(interesting_heads) - present_interesting_heads
460
if missing_interesting_heads:
461
raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)
462
return self._refs
463
464
465
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
466
def report_git_progress(pb, text):
467
text = text.rstrip("\r\n")
468
g = _GIT_PROGRESS_RE.match(text)
469
if g is not None:
470
(text, pct, current, total) = g.groups()
471
pb.update(text, int(current), int(total))
472
else:
473
pb.update(text, 0, 0)
474
475
476
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
477
"""InterRepository that copies revisions from a remote Git into a non-Git
478
repository."""
479
480
def get_target_heads(self):
481
# FIXME: This should be more efficient
482
all_revs = self.target.all_revision_ids()
483
parent_map = self.target.get_parent_map(all_revs)
484
all_parents = set()
485
map(all_parents.update, parent_map.itervalues())
486
return set(all_revs) - all_parents
487
488
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
187
def fetch_objects(self, determine_wants, mapping, pb=None):
489
188
def progress(text):
490
report_git_progress(pb, text)
491
store = BazaarObjectStore(self.target, mapping)
492
self.target.lock_write()
493
try:
494
heads = self.get_target_heads()
495
graph_walker = store.get_graph_walker(
496
[store._lookup_revision_sha1(head) for head in heads])
497
recorded_wants = []
498
499
def record_determine_wants(heads):
500
wants = determine_wants(heads)
501
recorded_wants.extend(wants)
502
return wants
503
504
create_pb = None
505
if pb is None:
506
create_pb = pb = ui.ui_factory.nested_progress_bar()
507
try:
508
objects_iter = self.source.fetch_objects(
509
record_determine_wants, graph_walker,
510
store.get_raw, progress)
511
return import_git_objects(self.target, mapping,
512
objects_iter, store, recorded_wants, pb, limit)
513
finally:
514
if create_pb:
515
create_pb.finished()
516
finally:
517
self.target.unlock()
518
519
@staticmethod
520
def is_compatible(source, target):
521
"""Be compatible with GitRepository."""
522
# FIXME: Also check target uses VersionedFile
523
return (isinstance(source, RemoteGitRepository) and
524
target.supports_rich_root() and
525
not isinstance(target, GitRepository))
526
527
528
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
529
"""InterRepository that copies revisions from a local Git into a non-Git
530
repository."""
531
532
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
533
"""Fetch objects.
534
"""
535
wants = determine_wants(self.source._git.get_refs())
189
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
190
graph_walker = BzrFetchGraphWalker(self.target, mapping)
536
191
create_pb = None
537
192
if pb is None:
538
193
create_pb = pb = ui.ui_factory.nested_progress_bar()
539
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
194
target_git_object_retriever = GitObjectConverter(self.target, mapping)
195
540
196
try:
541
197
self.target.lock_write()
542
198
try:
543
return import_git_objects(self.target, mapping,
544
self.source._git.object_store,
545
target_git_object_retriever, wants, pb, limit)
199
self.target.start_write_group()
200
try:
201
objects_iter = self.source.fetch_objects(determine_wants,
202
graph_walker,
203
target_git_object_retriever.__getitem__,
204
progress)
205
import_git_objects(self.target, mapping, objects_iter,
206
target_git_object_retriever, pb)
207
finally:
208
self.target.commit_write_group()
546
209
finally:
547
210
self.target.unlock()
548
211
finally:
549
212
if create_pb:
550
213
create_pb.finished()
551
214
215
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
216
mapping=None):
217
if mapping is None:
218
mapping = self.source.get_mapping()
219
def determine_wants(heads):
220
if revision_id is None:
221
ret = heads.values()
222
else:
223
ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
224
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
225
return self.fetch_objects(determine_wants, mapping, pb)
226
552
227
@staticmethod
553
228
def is_compatible(source, target):
554
229
"""Be compatible with GitRepository."""
555
230
# FIXME: Also check target uses VersionedFile
556
return (isinstance(source, LocalGitRepository) and
231
return (isinstance(source, GitRepository) and
557
232
target.supports_rich_root() and
558
233
not isinstance(target, GitRepository))
559
234
560
235
561
class InterGitGitRepository(InterGitRepository):
562
"""InterRepository that copies between Git repositories."""
563
564
def fetch_objects(self, determine_wants, mapping, pb=None):
236
class InterGitRepository(InterRepository):
237
238
_matching_repo_format = GitFormat()
239
240
@staticmethod
241
def _get_repo_format_to_test():
242
return None
243
244
def copy_content(self, revision_id=None, pb=None):
245
"""See InterRepository.copy_content."""
246
self.fetch(revision_id, pb, find_ghosts=False)
247
248
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
249
mapping=None):
250
if mapping is None:
251
mapping = self.source.get_mapping()
565
252
def progress(text):
566
trace.note("git: %s", text)
567
graphwalker = self.target._git.get_graph_walker()
568
if (isinstance(self.source, LocalGitRepository) and
569
isinstance(self.target, LocalGitRepository)):
570
return self.source._git.fetch(self.target._git, determine_wants,
571
progress)
572
elif (isinstance(self.source, LocalGitRepository) and
573
isinstance(self.target, RemoteGitRepository)):
574
raise NotImplementedError
575
elif (isinstance(self.source, RemoteGitRepository) and
576
isinstance(self.target, LocalGitRepository)):
577
f, commit = self.target._git.object_store.add_thin_pack()
578
try:
579
refs = self.source._git.fetch_pack(determine_wants,
580
graphwalker, f.write, progress)
581
commit()
582
return refs
583
except:
584
f.close()
585
raise
586
else:
587
raise AssertionError
588
589
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
590
mapping=None, fetch_spec=None, branches=None):
591
if mapping is None:
592
mapping = self.source.get_mapping()
253
info("git: %s", text)
593
254
r = self.target._git
594
if revision_id is not None:
255
if revision_id is None:
256
determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
257
else:
595
258
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
596
elif fetch_spec is not None:
597
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
598
if branches is not None:
599
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
600
elif fetch_spec is None and revision_id is None:
601
determine_wants = r.object_store.determine_wants_all
602
else:
603
259
determine_wants = lambda x: [y for y in args if not y in r.object_store]
604
return self.fetch_objects(determine_wants, mapping)[0]
605
260
261
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
262
f, commit = r.object_store.add_pack()
263
try:
264
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
265
f.close()
266
commit()
267
except:
268
f.close()
269
raise
606
270
607
271
@staticmethod
608
272
def is_compatible(source, target):
609
273
"""Be compatible with GitRepository."""
610
return (isinstance(source, GitRepository) and
274
return (isinstance(source, GitRepository) and
611
275
isinstance(target, GitRepository))
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0