To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.200.812
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.812
- Committer: Jelmer Vernooij
- Date: 2010-03-31 22:11:58 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100331221158-ywvpbv0dgwe50q2m
Catch KeyError from lookup_tree as well - some caches (such as sqlite) don't store all trees, only some.
- files added:
- HACKING
- files removed:
- foreign
- files renamed:
- converter.py => object_store.py
- tests/test_ids.py => tests/test_mapping.py
- files modified:
- Makefile
added
removed
fetch.py
1
# Copyright (C) 2008 Canonical Ltd
1
# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
23
17
from dulwich.objects import (
18
Commit,
19
Tag,
20
S_ISGITLINK,
21
)
22
from dulwich.object_store import (
23
tree_lookup_path,
24
)
25
import re
26
import stat
27
28
from bzrlib import (
29
debug,
30
osutils,
31
trace,
32
ui,
33
urlutils,
34
)
35
from bzrlib.errors import (
36
BzrError,
37
NoSuchId,
38
)
39
from bzrlib.inventory import (
40
Inventory,
41
InventoryDirectory,
42
InventoryFile,
43
InventoryLink,
44
TreeReference,
45
)
46
from bzrlib.repository import (
47
InterRepository,
48
)
49
from bzrlib.revision import (
50
NULL_REVISION,
51
)
52
from bzrlib.tsort import (
53
topo_sort,
54
)
55
from bzrlib.versionedfile import (
56
ChunkedContentFactory,
57
)
58
59
from bzrlib.plugins.git.mapping import (
60
DEFAULT_FILE_MODE,
61
inventory_to_tree_and_blobs,
62
mode_is_executable,
63
squash_revision,
64
warn_unusual_mode,
65
)
66
from bzrlib.plugins.git.object_store import (
67
BazaarObjectStore,
68
LRUInventoryCache,
69
)
70
from bzrlib.plugins.git.remote import (
71
RemoteGitRepository,
72
)
24
73
from bzrlib.plugins.git.repository import (
25
LocalGitRepository,
26
GitRepository,
27
GitFormat,
28
)
29
from bzrlib.plugins.git.converter import GitObjectConverter
30
from bzrlib.plugins.git.remote import RemoteGitRepository
31
32
import dulwich as git
33
from dulwich.client import SimpleFetchGraphWalker
34
from dulwich.objects import Commit
35
36
from cStringIO import StringIO
37
38
39
class BzrFetchGraphWalker(object):
40
"""GraphWalker implementation that uses a Bazaar repository."""
41
42
def __init__(self, repository, mapping):
43
self.repository = repository
44
self.mapping = mapping
45
self.done = set()
46
self.heads = set(repository.all_revision_ids())
47
self.parents = {}
48
49
def __iter__(self):
50
return iter(self.next, None)
51
52
def ack(self, sha):
53
revid = self.mapping.revision_id_foreign_to_bzr(sha)
54
self.remove(revid)
55
56
def remove(self, revid):
57
self.done.add(revid)
58
if revid in self.heads:
59
self.heads.remove(revid)
60
if revid in self.parents:
61
for p in self.parents[revid]:
62
self.remove(p)
63
64
def next(self):
65
while self.heads:
66
ret = self.heads.pop()
67
ps = self.repository.get_parent_map([ret])[ret]
68
self.parents[ret] = ps
69
self.heads.update([p for p in ps if not p in self.done])
70
try:
71
self.done.add(ret)
72
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
73
except InvalidRevisionId:
74
pass
75
return None
76
77
78
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, gitmap, executable):
74
GitRepository,
75
GitRepositoryFormat,
76
LocalGitRepository,
77
)
78
79
80
def import_git_blob(texts, mapping, path, hexsha, base_inv, base_inv_shamap,
81
base_ie, parent_id, revision_id, parent_invs, lookup_object,
82
executable, symlink):
79
83
"""Import a git blob object into a bzr repository.
80
84
81
:param repo: bzr repository
85
:param texts: VersionedFiles to add to
82
86
:param path: Path in the tree
83
87
:param blob: A git blob
88
:return: Inventory delta for this file
84
89
"""
85
90
file_id = mapping.generate_file_id(path)
86
text_revision = inv.revision_id
87
repo.texts.add_lines((file_id, text_revision),
88
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
89
osutils.split_lines(blob.data))
90
ie = inv.add_path(path, "file", file_id)
91
ie.revision = text_revision
92
ie.text_size = len(blob.data)
93
ie.text_sha1 = osutils.sha_string(blob.data)
91
if symlink:
92
cls = InventoryLink
93
else:
94
cls = InventoryFile
95
# We just have to hope this is indeed utf-8:
96
ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)
94
97
ie.executable = executable
95
gitmap._idmap.add_entry(blob.sha().hexdigest(), "blob", (ie.file_id, ie.revision))
96
97
98
def import_git_tree(repo, mapping, path, tree, inv, parent_invs,
99
gitmap, lookup_object):
98
# See if this has changed at all
99
if base_ie is None:
100
base_sha = None
101
else:
102
try:
103
base_sha = base_inv_shamap.lookup_blob(file_id, base_ie.revision)
104
except KeyError:
105
base_sha = None
106
else:
107
if (base_sha == hexsha and base_ie.executable == ie.executable
108
and base_ie.kind == ie.kind):
109
# If nothing has changed since the base revision, we're done
110
return [], []
111
if base_sha == hexsha and base_ie.kind == ie.kind:
112
ie.text_size = base_ie.text_size
113
ie.text_sha1 = base_ie.text_sha1
114
ie.symlink_target = base_ie.symlink_target
115
if ie.executable == base_ie.executable:
116
ie.revision = base_ie.revision
117
else:
118
blob = lookup_object(hexsha)
119
else:
120
blob = lookup_object(hexsha)
121
if ie.kind == "symlink":
122
ie.revision = None
123
ie.symlink_target = blob.data
124
ie.text_size = None
125
ie.text_sha1 = None
126
else:
127
ie.text_size = len(blob.data)
128
ie.text_sha1 = osutils.sha_string(blob.data)
129
# Check what revision we should store
130
parent_keys = []
131
for pinv in parent_invs:
132
if pinv.revision_id == base_inv.revision_id:
133
pie = base_ie
134
if pie is None:
135
continue
136
else:
137
try:
138
pie = pinv[file_id]
139
except NoSuchId:
140
continue
141
if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:
142
# found a revision in one of the parents to use
143
ie.revision = pie.revision
144
break
145
parent_keys.append((file_id, pie.revision))
146
if ie.revision is None:
147
# Need to store a new revision
148
ie.revision = revision_id
149
assert file_id is not None
150
assert ie.revision is not None
151
if ie.kind == 'symlink':
152
chunks = []
153
else:
154
try:
155
chunks = blob.chunked
156
except AttributeError: # older version of dulwich
157
chunks = [blob.data]
158
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, chunks)])
159
shamap = { ie.file_id: hexsha }
160
invdelta = []
161
if base_ie is not None:
162
old_path = base_inv.id2path(file_id)
163
if base_ie.kind == "directory":
164
invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))
165
else:
166
old_path = None
167
invdelta.append((old_path, path, file_id, ie))
168
return (invdelta, shamap)
169
170
171
class SubmodulesRequireSubtrees(BzrError):
172
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
173
internal = False
174
175
176
def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,
177
parent_id, revision_id, parent_invs, lookup_object):
178
file_id = mapping.generate_file_id(path)
179
ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),
180
parent_id)
181
ie.revision = revision_id
182
if base_ie is None:
183
oldpath = None
184
else:
185
oldpath = path
186
if (base_ie.kind == ie.kind and
187
base_ie.reference_revision == ie.reference_revision):
188
ie.revision = base_ie.revision
189
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
190
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])
191
invdelta = [(oldpath, path, file_id, ie)]
192
return invdelta, {}, {}
193
194
195
def remove_disappeared_children(path, base_children, existing_children):
196
ret = []
197
deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]
198
while deletable:
199
(path, ie) = deletable.pop()
200
ret.append((path, None, ie.file_id, None))
201
if ie.kind == "directory":
202
for name, child_ie in ie.children.iteritems():
203
deletable.append((osutils.pathjoin(path, name), child_ie))
204
return ret
205
206
207
def import_git_tree(texts, mapping, path, hexsha, base_inv, base_inv_shamap,
208
base_ie, parent_id, revision_id, parent_invs, lookup_object,
209
allow_submodules=False):
100
210
"""Import a git tree object into a bzr repository.
101
211
102
:param repo: A Bzr repository object
212
:param texts: VersionedFiles object to add to
103
213
:param path: Path in the tree
104
214
:param tree: A git tree object
105
:param inv: Inventory object
215
:param base_inv: Base inventory against which to return inventory delta
216
:return: Inventory delta for this subtree
106
217
"""
218
invdelta = []
107
219
file_id = mapping.generate_file_id(path)
108
text_revision = inv.revision_id
109
repo.texts.add_lines((file_id, text_revision),
110
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
111
[])
112
ie = inv.add_path(path, "directory", file_id)
113
ie.revision = text_revision
114
gitmap._idmap.add_entry(tree.sha().hexdigest(), "tree", (file_id, text_revision))
115
for mode, name, hexsha in tree.entries():
116
entry_kind = (mode & 0700000) / 0100000
220
# We just have to hope this is indeed utf-8:
221
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
222
parent_id)
223
if base_ie is None:
224
# Newly appeared here
225
ie.revision = revision_id
226
texts.insert_record_stream([ChunkedContentFactory((file_id, ie.revision), (), None, [])])
227
invdelta.append((None, path, file_id, ie))
228
else:
229
# See if this has changed at all
230
try:
231
base_sha = base_inv_shamap.lookup_tree(file_id)
232
except (KeyError, NotImplementedError):
233
pass
234
else:
235
if base_sha == hexsha:
236
# If nothing has changed since the base revision, we're done
237
return [], {}, []
238
if base_ie.kind != "directory":
239
ie.revision = revision_id
240
texts.insert_record_stream([ChunkedContentFactory((ie.file_id, ie.revision), (), None, [])])
241
invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))
242
if base_ie is not None and base_ie.kind == "directory":
243
base_children = base_ie.children
244
else:
245
base_children = {}
246
# Remember for next time
247
existing_children = set()
248
child_modes = {}
249
shamap = {}
250
tree = lookup_object(hexsha)
251
for mode, name, child_hexsha in tree.entries():
117
252
basename = name.decode("utf-8")
118
if path == "":
119
child_path = name
120
else:
121
child_path = urlutils.join(path, name)
122
if entry_kind == 0:
123
tree = lookup_object(hexsha)
124
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, gitmap, lookup_object)
125
elif entry_kind == 1:
126
blob = lookup_object(hexsha)
127
fs_mode = mode & 0777
128
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, gitmap, bool(fs_mode & 0111))
129
else:
130
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
131
132
133
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
134
pb=None):
253
existing_children.add(basename)
254
child_path = osutils.pathjoin(path, name)
255
if stat.S_ISDIR(mode):
256
subinvdelta, grandchildmodes, subshamap = import_git_tree(
257
texts, mapping, child_path, child_hexsha, base_inv,
258
base_inv_shamap, base_children.get(basename), file_id,
259
revision_id, parent_invs, lookup_object,
260
allow_submodules=allow_submodules)
261
elif S_ISGITLINK(mode): # submodule
262
if not allow_submodules:
263
raise SubmodulesRequireSubtrees()
264
subinvdelta, grandchildmodes, subshamap = import_git_submodule(
265
texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),
266
file_id, revision_id, parent_invs, lookup_object)
267
else:
268
subinvdelta, subshamap = import_git_blob(texts, mapping,
269
child_path, child_hexsha, base_inv, base_inv_shamap,
270
base_children.get(basename), file_id,
271
revision_id, parent_invs, lookup_object,
272
mode_is_executable(mode), stat.S_ISLNK(mode))
273
grandchildmodes = {}
274
child_modes.update(grandchildmodes)
275
invdelta.extend(subinvdelta)
276
shamap.update(subshamap)
277
if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
278
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
279
child_modes[child_path] = mode
280
# Remove any children that have disappeared
281
if base_ie is not None and base_ie.kind == "directory":
282
invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),
283
base_children, existing_children))
284
shamap[file_id] = hexsha
285
return invdelta, child_modes, shamap
286
287
288
def import_git_commit(repo, mapping, head, lookup_object,
289
target_git_object_retriever, parent_invs_cache):
290
o = lookup_object(head)
291
rev = mapping.import_commit(o)
292
# We have to do this here, since we have to walk the tree and
293
# we need to make sure to import the blobs / trees with the right
294
# path; this may involve adding them more than once.
295
parent_invs = parent_invs_cache.get_inventories(rev.parent_ids)
296
if parent_invs == []:
297
base_inv = Inventory(root_id=None)
298
base_ie = None
299
base_inv_shamap = None # Should never be accessed
300
else:
301
base_inv = parent_invs[0]
302
base_ie = base_inv.root
303
base_inv_shamap = target_git_object_retriever._idmap.get_inventory_sha_map(base_inv.revision_id)
304
inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,
305
mapping, "", o.tree, base_inv, base_inv_shamap, base_ie, None,
306
rev.revision_id, parent_invs, lookup_object,
307
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
308
entries = []
309
for (oldpath, newpath, fileid, new_ie) in inv_delta:
310
if newpath is None:
311
entries.append((fileid, None, None, None))
312
else:
313
if new_ie.kind in ("file", "symlink"):
314
entries.append((fileid, "blob", shamap[fileid], new_ie.revision))
315
elif new_ie.kind == "directory":
316
entries.append((fileid, "tree", shamap[fileid], rev.revision_id))
317
else:
318
raise AssertionError
319
target_git_object_retriever._idmap.add_entries(rev.revision_id,
320
rev.parent_ids, head, o.tree, entries)
321
if unusual_modes != {}:
322
for path, mode in unusual_modes.iteritems():
323
warn_unusual_mode(rev.foreign_revid, path, mode)
324
mapping.import_unusual_file_modes(rev, unusual_modes)
325
try:
326
basis_id = rev.parent_ids[0]
327
except IndexError:
328
basis_id = NULL_REVISION
329
base_inv = None
330
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
331
inv_delta, rev.revision_id, rev.parent_ids,
332
base_inv)
333
parent_invs_cache.add(rev.revision_id, inv)
334
repo.add_revision(rev.revision_id, rev)
335
if "verify" in debug.debug_flags:
336
new_unusual_modes = mapping.export_unusual_file_modes(rev)
337
if new_unusual_modes != unusual_modes:
338
raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))
339
objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)
340
for newsha1, newobj, path in objs:
341
assert path is not None
342
if path == "":
343
oldsha1 = o.tree
344
else:
345
(oldmode, oldsha1) = tree_lookup_path(lookup_object, o.tree, path)
346
if oldsha1 != newsha1:
347
raise AssertionError("%r != %r in %s" % (oldsha1, newsha1, path))
348
349
350
def import_git_objects(repo, mapping, object_iter,
351
target_git_object_retriever, heads, pb=None, limit=None):
135
352
"""Import a set of git objects into a bzr repository.
136
353
137
:param repo: Bazaar repository
354
:param repo: Target Bazaar repository
138
355
:param mapping: Mapping to use
139
356
:param object_iter: Iterator over Git objects.
357
:return: Tuple with pack hints and last imported revision id
140
358
"""
359
def lookup_object(sha):
360
try:
361
return object_iter[sha]
362
except KeyError:
363
return target_git_object_retriever[sha]
141
364
# TODO: a more (memory-)efficient implementation of this
142
365
graph = []
143
root_trees = {}
144
revisions = {}
366
checked = set()
367
heads = list(set(heads))
368
parent_invs_cache = LRUInventoryCache(repo)
369
target_git_object_retriever.start_write_group() # FIXME: try/finally
145
370
# Find and convert commit objects
146
for o in object_iter.iterobjects():
371
while heads:
372
if pb is not None:
373
pb.update("finding revisions to fetch", len(graph), None)
374
head = heads.pop()
375
assert isinstance(head, str)
376
try:
377
o = lookup_object(head)
378
except KeyError:
379
continue
147
380
if isinstance(o, Commit):
148
381
rev = mapping.import_commit(o)
149
root_trees[rev.revision_id] = object_iter[o.tree]
150
revisions[rev.revision_id] = rev
151
graph.append((rev.revision_id, rev.parent_ids))
152
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(), "commit", (rev.revision_id, o._tree))
382
if repo.has_revision(rev.revision_id):
383
continue
384
squash_revision(repo, rev)
385
graph.append((o.id, o.parents))
386
heads.extend([p for p in o.parents if p not in checked])
387
elif isinstance(o, Tag):
388
if o.object[1] not in checked:
389
heads.append(o.object[1])
390
else:
391
trace.warning("Unable to import head object %r" % o)
392
checked.add(o.id)
393
del checked
153
394
# Order the revisions
154
395
# Create the inventory objects
155
for i, revid in enumerate(topo_sort(graph)):
156
if pb is not None:
157
pb.update("fetching revisions", i, len(graph))
158
root_tree = root_trees[revid]
159
rev = revisions[revid]
160
# We have to do this here, since we have to walk the tree and
161
# we need to make sure to import the blobs / trees with the riht
162
# path; this may involve adding them more than once.
163
inv = Inventory()
164
inv.revision_id = rev.revision_id
165
def lookup_object(sha):
166
if sha in object_iter:
167
return object_iter[sha]
168
return target_git_object_retriever[sha]
169
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
170
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
171
target_git_object_retriever, lookup_object)
172
repo.add_revision(rev.revision_id, rev, inv)
173
174
175
class InterGitNonGitRepository(InterRepository):
176
177
_matching_repo_format = GitFormat()
396
batch_size = 100
397
revision_ids = topo_sort(graph)
398
pack_hints = []
399
if limit is not None:
400
revision_ids = revision_ids[:limit]
401
last_imported = None
402
for offset in range(0, len(revision_ids), batch_size):
403
repo.start_write_group()
404
try:
405
for i, head in enumerate(revision_ids[offset:offset+batch_size]):
406
if pb is not None:
407
pb.update("fetching revisions", offset+i, len(revision_ids))
408
import_git_commit(repo, mapping, head, lookup_object,
409
target_git_object_retriever,
410
parent_invs_cache)
411
last_imported = head
412
except:
413
repo.abort_write_group()
414
raise
415
else:
416
hint = repo.commit_write_group()
417
if hint is not None:
418
pack_hints.extend(hint)
419
target_git_object_retriever.commit_write_group()
420
return pack_hints, last_imported
421
422
423
class InterGitRepository(InterRepository):
424
425
_matching_repo_format = GitRepositoryFormat()
178
426
179
427
@staticmethod
180
428
def _get_repo_format_to_test():
184
432
"""See InterRepository.copy_content."""
185
433
self.fetch(revision_id, pb, find_ghosts=False)
186
434
187
def fetch_objects(self, determine_wants, mapping, pb=None):
435
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
436
mapping=None, fetch_spec=None):
437
self.fetch_refs(revision_id=revision_id, pb=pb,
438
find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)
439
440
441
class InterGitNonGitRepository(InterGitRepository):
442
"""Base InterRepository that copies revisions from a Git into a non-Git
443
repository."""
444
445
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
446
mapping=None, fetch_spec=None):
447
if mapping is None:
448
mapping = self.source.get_mapping()
449
if revision_id is not None:
450
interesting_heads = [revision_id]
451
elif fetch_spec is not None:
452
interesting_heads = fetch_spec.heads
453
else:
454
interesting_heads = None
455
self._refs = {}
456
def determine_wants(refs):
457
self._refs = refs
458
if interesting_heads is None:
459
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
460
else:
461
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
462
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
463
(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)
464
if pack_hint is not None and self.target._format.pack_compresses:
465
self.target.pack(hint=pack_hint)
466
if interesting_heads is not None:
467
present_interesting_heads = self.target.has_revisions(interesting_heads)
468
missing_interesting_heads = set(interesting_heads) - present_interesting_heads
469
if missing_interesting_heads:
470
raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)
471
return self._refs
472
473
474
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
475
def report_git_progress(pb, text):
476
text = text.rstrip("\r\n")
477
g = _GIT_PROGRESS_RE.match(text)
478
if g is not None:
479
(text, pct, current, total) = g.groups()
480
pb.update(text, int(current), int(total))
481
else:
482
pb.update(text, 0, 0)
483
484
485
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
486
"""InterRepository that copies revisions from a remote Git into a non-Git
487
repository."""
488
489
def get_target_heads(self):
490
# FIXME: This should be more efficient
491
all_revs = self.target.all_revision_ids()
492
parent_map = self.target.get_parent_map(all_revs)
493
all_parents = set()
494
map(all_parents.update, parent_map.itervalues())
495
return set(all_revs) - all_parents
496
497
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
188
498
def progress(text):
189
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
190
graph_walker = BzrFetchGraphWalker(self.target, mapping)
499
report_git_progress(pb, text)
500
store = BazaarObjectStore(self.target, mapping)
501
self.target.lock_write()
502
try:
503
heads = self.get_target_heads()
504
graph_walker = store.get_graph_walker(
505
[store._lookup_revision_sha1(head) for head in heads])
506
recorded_wants = []
507
508
def record_determine_wants(heads):
509
wants = determine_wants(heads)
510
recorded_wants.extend(wants)
511
return wants
512
513
create_pb = None
514
if pb is None:
515
create_pb = pb = ui.ui_factory.nested_progress_bar()
516
try:
517
objects_iter = self.source.fetch_objects(
518
record_determine_wants, graph_walker,
519
store.get_raw, progress)
520
return import_git_objects(self.target, mapping,
521
objects_iter, store, recorded_wants, pb, limit)
522
finally:
523
if create_pb:
524
create_pb.finished()
525
finally:
526
self.target.unlock()
527
528
@staticmethod
529
def is_compatible(source, target):
530
"""Be compatible with GitRepository."""
531
# FIXME: Also check target uses VersionedFile
532
return (isinstance(source, RemoteGitRepository) and
533
target.supports_rich_root() and
534
not isinstance(target, GitRepository))
535
536
537
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
538
"""InterRepository that copies revisions from a local Git into a non-Git
539
repository."""
540
541
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
542
"""Fetch objects.
543
"""
544
wants = determine_wants(self.source._git.get_refs())
191
545
create_pb = None
192
546
if pb is None:
193
547
create_pb = pb = ui.ui_factory.nested_progress_bar()
194
target_git_object_retriever = GitObjectConverter(self.target, mapping)
195
548
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
196
549
try:
197
550
self.target.lock_write()
198
551
try:
199
self.target.start_write_group()
200
try:
201
objects_iter = self.source.fetch_objects(determine_wants,
202
graph_walker,
203
target_git_object_retriever.__getitem__,
204
progress)
205
import_git_objects(self.target, mapping, objects_iter,
206
target_git_object_retriever, pb)
207
finally:
208
self.target.commit_write_group()
552
return import_git_objects(self.target, mapping,
553
self.source._git.object_store,
554
target_git_object_retriever, wants, pb, limit)
209
555
finally:
210
556
self.target.unlock()
211
557
finally:
212
558
if create_pb:
213
559
create_pb.finished()
214
560
215
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
216
mapping=None):
217
if mapping is None:
218
mapping = self.source.get_mapping()
219
def determine_wants(heads):
220
if revision_id is None:
221
ret = heads.values()
222
else:
223
ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
224
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
225
return self.fetch_objects(determine_wants, mapping, pb)
226
227
561
@staticmethod
228
562
def is_compatible(source, target):
229
563
"""Be compatible with GitRepository."""
230
564
# FIXME: Also check target uses VersionedFile
231
return (isinstance(source, GitRepository) and
565
return (isinstance(source, LocalGitRepository) and
232
566
target.supports_rich_root() and
233
567
not isinstance(target, GitRepository))
234
568
235
569
236
class InterGitRepository(InterRepository):
237
238
_matching_repo_format = GitFormat()
239
240
@staticmethod
241
def _get_repo_format_to_test():
242
return None
243
244
def copy_content(self, revision_id=None, pb=None):
245
"""See InterRepository.copy_content."""
246
self.fetch(revision_id, pb, find_ghosts=False)
247
248
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
249
mapping=None):
250
if mapping is None:
251
mapping = self.source.get_mapping()
570
class InterGitGitRepository(InterGitRepository):
571
"""InterRepository that copies between Git repositories."""
572
573
def fetch_objects(self, determine_wants, mapping, pb=None):
252
574
def progress(text):
253
info("git: %s", text)
575
trace.note("git: %s", text)
576
graphwalker = self.target._git.get_graph_walker()
577
if (isinstance(self.source, LocalGitRepository) and
578
isinstance(self.target, LocalGitRepository)):
579
return self.source._git.fetch(self.target._git, determine_wants,
580
progress)
581
elif (isinstance(self.source, LocalGitRepository) and
582
isinstance(self.target, RemoteGitRepository)):
583
raise NotImplementedError
584
elif (isinstance(self.source, RemoteGitRepository) and
585
isinstance(self.target, LocalGitRepository)):
586
f, commit = self.target._git.object_store.add_thin_pack()
587
try:
588
refs = self.source._git.fetch_pack(determine_wants,
589
graphwalker, f.write, progress)
590
commit()
591
return refs
592
except:
593
f.close()
594
raise
595
else:
596
raise AssertionError
597
598
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
599
mapping=None, fetch_spec=None, branches=None):
600
if mapping is None:
601
mapping = self.source.get_mapping()
254
602
r = self.target._git
255
if revision_id is None:
256
determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
257
else:
603
if revision_id is not None:
258
604
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
605
elif fetch_spec is not None:
606
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
607
if branches is not None:
608
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
609
elif fetch_spec is None and revision_id is None:
610
determine_wants = r.object_store.determine_wants_all
611
else:
259
612
determine_wants = lambda x: [y for y in args if not y in r.object_store]
613
return self.fetch_objects(determine_wants, mapping)[0]
260
614
261
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
262
f, commit = r.object_store.add_pack()
263
try:
264
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
265
f.close()
266
commit()
267
except:
268
f.close()
269
raise
270
615
271
616
@staticmethod
272
617
def is_compatible(source, target):
273
618
"""Be compatible with GitRepository."""
274
return (isinstance(source, GitRepository) and
619
return (isinstance(source, GitRepository) and
275
620
isinstance(target, GitRepository))
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0