To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.248.6
- compare with another revision
- download diff
- download tarball
- view history from revision 0.248.6
- Committer: Jelmer Vernooij
- Date: 2010-03-25 11:29:44 UTC
- mto: (0.254.20 index-based)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100325112944-yx5r5hbvlzlswi1e
Mark as compatible with Bazaar 2.2
- files added:
- HACKING
- files removed:
- foreign
- files renamed:
- converter.py => object_store.py
- tests/test_ids.py => tests/test_mapping.py
- files modified:
- Makefile
added
removed
fetch.py
1
# Copyright (C) 2008 Canonical Ltd
1
# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
23
17
from dulwich.objects import (
18
Commit,
19
Tag,
20
S_ISGITLINK,
21
)
22
from dulwich.object_store import (
23
tree_lookup_path,
24
)
25
import re
26
import stat
27
28
from bzrlib import (
29
debug,
30
lru_cache,
31
osutils,
32
trace,
33
ui,
34
urlutils,
35
)
36
from bzrlib.errors import (
37
BzrError,
38
NoSuchId,
39
)
40
from bzrlib.inventory import (
41
Inventory,
42
InventoryDirectory,
43
InventoryFile,
44
InventoryLink,
45
TreeReference,
46
)
47
from bzrlib.repository import (
48
InterRepository,
49
)
50
from bzrlib.revision import (
51
NULL_REVISION,
52
)
53
from bzrlib.tsort import (
54
topo_sort,
55
)
56
from bzrlib.versionedfile import (
57
FulltextContentFactory,
58
)
59
60
from bzrlib.plugins.git.mapping import (
61
DEFAULT_FILE_MODE,
62
inventory_to_tree_and_blobs,
63
mode_is_executable,
64
squash_revision,
65
warn_unusual_mode,
66
)
67
from bzrlib.plugins.git.object_store import (
68
BazaarObjectStore,
69
)
70
from bzrlib.plugins.git.remote import (
71
RemoteGitRepository,
72
)
24
73
from bzrlib.plugins.git.repository import (
25
LocalGitRepository,
26
GitRepository,
27
GitFormat,
28
)
29
from bzrlib.plugins.git.converter import GitObjectConverter
30
from bzrlib.plugins.git.remote import RemoteGitRepository
31
32
import dulwich as git
33
from dulwich.client import SimpleFetchGraphWalker
34
from dulwich.objects import Commit
35
36
from cStringIO import StringIO
37
38
39
class BzrFetchGraphWalker(object):
40
"""GraphWalker implementation that uses a Bazaar repository."""
41
42
def __init__(self, repository, mapping):
43
self.repository = repository
44
self.mapping = mapping
45
self.done = set()
46
self.heads = set(repository.all_revision_ids())
47
self.parents = {}
48
49
def __iter__(self):
50
return iter(self.next, None)
51
52
def ack(self, sha):
53
revid = self.mapping.revision_id_foreign_to_bzr(sha)
54
self.remove(revid)
55
56
def remove(self, revid):
57
self.done.add(revid)
58
if revid in self.heads:
59
self.heads.remove(revid)
60
if revid in self.parents:
61
for p in self.parents[revid]:
62
self.remove(p)
63
64
def next(self):
65
while self.heads:
66
ret = self.heads.pop()
67
ps = self.repository.get_parent_map([ret])[ret]
68
self.parents[ret] = ps
69
self.heads.update([p for p in ps if not p in self.done])
70
try:
71
self.done.add(ret)
72
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
73
except InvalidRevisionId:
74
pass
75
return None
76
77
78
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, gitmap, executable):
74
GitRepository,
75
GitRepositoryFormat,
76
LocalGitRepository,
77
)
78
79
80
MAX_INV_CACHE_SIZE = 50 * 1024 * 1024
81
82
83
def import_git_blob(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,
84
revision_id, parent_invs, shagitmap, lookup_object, executable, symlink):
79
85
"""Import a git blob object into a bzr repository.
80
86
81
:param repo: bzr repository
87
:param texts: VersionedFiles to add to
82
88
:param path: Path in the tree
83
89
:param blob: A git blob
90
:return: Inventory delta for this file
84
91
"""
85
92
file_id = mapping.generate_file_id(path)
86
text_revision = inv.revision_id
87
repo.texts.add_lines((file_id, text_revision),
88
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
89
osutils.split_lines(blob.data))
90
ie = inv.add_path(path, "file", file_id)
91
ie.revision = text_revision
92
ie.text_size = len(blob.data)
93
ie.text_sha1 = osutils.sha_string(blob.data)
93
if symlink:
94
cls = InventoryLink
95
else:
96
cls = InventoryFile
97
# We just have to hope this is indeed utf-8:
98
ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)
94
99
ie.executable = executable
95
gitmap._idmap.add_entry(blob.sha().hexdigest(), "blob", (ie.file_id, ie.revision))
96
97
98
def import_git_tree(repo, mapping, path, tree, inv, parent_invs,
99
gitmap, lookup_object):
100
# See if this has changed at all
101
if base_ie is None:
102
base_sha = None
103
else:
104
try:
105
base_sha = shagitmap.lookup_blob(file_id, base_ie.revision)
106
except KeyError:
107
base_sha = None
108
else:
109
if (base_sha == hexsha and base_ie.executable == ie.executable
110
and base_ie.kind == ie.kind):
111
# If nothing has changed since the base revision, we're done
112
return [], []
113
if base_sha == hexsha and base_ie.kind == ie.kind:
114
ie.text_size = base_ie.text_size
115
ie.text_sha1 = base_ie.text_sha1
116
ie.symlink_target = base_ie.symlink_target
117
if ie.executable == base_ie.executable:
118
ie.revision = base_ie.revision
119
else:
120
blob = lookup_object(hexsha)
121
else:
122
blob = lookup_object(hexsha)
123
if ie.kind == "symlink":
124
ie.revision = None
125
ie.symlink_target = blob.data
126
ie.text_size = None
127
ie.text_sha1 = None
128
else:
129
ie.text_size = len(blob.data)
130
ie.text_sha1 = osutils.sha_string(blob.data)
131
# Check what revision we should store
132
parent_keys = []
133
for pinv in parent_invs:
134
if pinv.revision_id == base_inv.revision_id:
135
pie = base_ie
136
if pie is None:
137
continue
138
else:
139
try:
140
pie = pinv[file_id]
141
except NoSuchId:
142
continue
143
if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:
144
# found a revision in one of the parents to use
145
ie.revision = pie.revision
146
break
147
parent_keys.append((file_id, pie.revision))
148
if ie.revision is None:
149
# Need to store a new revision
150
ie.revision = revision_id
151
assert file_id is not None
152
assert ie.revision is not None
153
if ie.kind == 'symlink':
154
data = ''
155
else:
156
data = blob.data
157
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, data)])
158
shamap = [(hexsha, "blob", (ie.file_id, ie.revision))]
159
else:
160
shamap = []
161
invdelta = []
162
if base_ie is not None:
163
old_path = base_inv.id2path(file_id)
164
if base_ie.kind == "directory":
165
invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))
166
else:
167
old_path = None
168
invdelta.append((old_path, path, file_id, ie))
169
return (invdelta, shamap)
170
171
172
class SubmodulesRequireSubtrees(BzrError):
173
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
174
internal = False
175
176
177
def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,
178
parent_id, revision_id, parent_invs, shagitmap, lookup_object):
179
file_id = mapping.generate_file_id(path)
180
ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),
181
parent_id)
182
ie.revision = revision_id
183
if base_ie is None:
184
oldpath = None
185
else:
186
oldpath = path
187
if (base_ie.kind == ie.kind and
188
base_ie.reference_revision == ie.reference_revision):
189
ie.revision = base_ie.revision
190
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
191
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
192
invdelta = [(oldpath, path, file_id, ie)]
193
return invdelta, {}, {}
194
195
196
def remove_disappeared_children(path, base_children, existing_children):
197
ret = []
198
deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]
199
while deletable:
200
(path, ie) = deletable.pop()
201
ret.append((path, None, ie.file_id, None))
202
if ie.kind == "directory":
203
for name, child_ie in ie.children.iteritems():
204
deletable.append((osutils.pathjoin(path, name), child_ie))
205
return ret
206
207
208
def import_git_tree(texts, mapping, path, hexsha, base_inv, base_ie, parent_id,
209
revision_id, parent_invs, shagitmap, lookup_object, allow_submodules=False):
100
210
"""Import a git tree object into a bzr repository.
101
211
102
:param repo: A Bzr repository object
212
:param texts: VersionedFiles object to add to
103
213
:param path: Path in the tree
104
214
:param tree: A git tree object
105
:param inv: Inventory object
215
:param base_inv: Base inventory against which to return inventory delta
216
:return: Inventory delta for this subtree
106
217
"""
218
invdelta = []
107
219
file_id = mapping.generate_file_id(path)
108
text_revision = inv.revision_id
109
repo.texts.add_lines((file_id, text_revision),
110
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
111
[])
112
ie = inv.add_path(path, "directory", file_id)
113
ie.revision = text_revision
114
gitmap._idmap.add_entry(tree.sha().hexdigest(), "tree", (file_id, text_revision))
115
for mode, name, hexsha in tree.entries():
116
entry_kind = (mode & 0700000) / 0100000
220
# We just have to hope this is indeed utf-8:
221
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
222
parent_id)
223
if base_ie is None:
224
# Newly appeared here
225
ie.revision = revision_id
226
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
227
invdelta.append((None, path, file_id, ie))
228
else:
229
# See if this has changed at all
230
try:
231
base_sha = shagitmap.lookup_tree(file_id, base_inv.revision_id)
232
except KeyError:
233
pass
234
else:
235
if base_sha == hexsha:
236
# If nothing has changed since the base revision, we're done
237
return [], {}, []
238
if base_ie.kind != "directory":
239
ie.revision = revision_id
240
texts.insert_record_stream([FulltextContentFactory((ie.file_id, ie.revision), (), None, "")])
241
invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))
242
if base_ie is not None and base_ie.kind == "directory":
243
base_children = base_ie.children
244
else:
245
base_children = {}
246
# Remember for next time
247
existing_children = set()
248
child_modes = {}
249
shamap = []
250
tree = lookup_object(hexsha)
251
for mode, name, child_hexsha in tree.entries():
117
252
basename = name.decode("utf-8")
118
if path == "":
119
child_path = name
120
else:
121
child_path = urlutils.join(path, name)
122
if entry_kind == 0:
123
tree = lookup_object(hexsha)
124
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, gitmap, lookup_object)
125
elif entry_kind == 1:
126
blob = lookup_object(hexsha)
127
fs_mode = mode & 0777
128
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, gitmap, bool(fs_mode & 0111))
129
else:
130
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
131
132
133
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
134
pb=None):
253
existing_children.add(basename)
254
child_path = osutils.pathjoin(path, name)
255
if stat.S_ISDIR(mode):
256
subinvdelta, grandchildmodes, subshamap = import_git_tree(
257
texts, mapping, child_path, child_hexsha, base_inv,
258
base_children.get(basename), file_id, revision_id,
259
parent_invs, shagitmap, lookup_object,
260
allow_submodules=allow_submodules)
261
invdelta.extend(subinvdelta)
262
child_modes.update(grandchildmodes)
263
shamap.extend(subshamap)
264
elif S_ISGITLINK(mode): # submodule
265
if not allow_submodules:
266
raise SubmodulesRequireSubtrees()
267
subinvdelta, grandchildmodes, subshamap = import_git_submodule(
268
texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),
269
file_id, revision_id, parent_invs, shagitmap, lookup_object)
270
invdelta.extend(subinvdelta)
271
child_modes.update(grandchildmodes)
272
shamap.extend(subshamap)
273
else:
274
subinvdelta, subshamap = import_git_blob(texts, mapping,
275
child_path, child_hexsha, base_inv, base_children.get(basename), file_id,
276
revision_id, parent_invs, shagitmap, lookup_object,
277
mode_is_executable(mode), stat.S_ISLNK(mode))
278
invdelta.extend(subinvdelta)
279
shamap.extend(subshamap)
280
if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
281
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
282
child_modes[child_path] = mode
283
# Remove any children that have disappeared
284
if base_ie is not None and base_ie.kind == "directory":
285
invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),
286
base_children, existing_children))
287
shamap.append((hexsha, "tree", (file_id, revision_id)))
288
return invdelta, child_modes, shamap
289
290
291
def approx_inv_size(inv):
292
# Very rough estimate, 1k per inventory entry
293
return len(inv) * 1024
294
295
296
def import_git_commit(repo, mapping, head, lookup_object,
297
target_git_object_retriever, parent_invs_cache):
298
o = lookup_object(head)
299
rev = mapping.import_commit(o)
300
# We have to do this here, since we have to walk the tree and
301
# we need to make sure to import the blobs / trees with the right
302
# path; this may involve adding them more than once.
303
parent_invs = []
304
for parent_id in rev.parent_ids:
305
try:
306
parent_invs.append(parent_invs_cache[parent_id])
307
except KeyError:
308
parent_inv = repo.get_inventory(parent_id)
309
parent_invs.append(parent_inv)
310
parent_invs_cache[parent_id] = parent_inv
311
if parent_invs == []:
312
base_inv = Inventory(root_id=None)
313
base_ie = None
314
else:
315
base_inv = parent_invs[0]
316
base_ie = base_inv.root
317
inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,
318
mapping, "", o.tree, base_inv, base_ie, None, rev.revision_id,
319
parent_invs, target_git_object_retriever._idmap, lookup_object,
320
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
321
target_git_object_retriever._idmap.add_entries(shamap)
322
if unusual_modes != {}:
323
for path, mode in unusual_modes.iteritems():
324
warn_unusual_mode(rev.foreign_revid, path, mode)
325
mapping.import_unusual_file_modes(rev, unusual_modes)
326
try:
327
basis_id = rev.parent_ids[0]
328
except IndexError:
329
basis_id = NULL_REVISION
330
base_inv = None
331
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
332
inv_delta, rev.revision_id, rev.parent_ids,
333
base_inv)
334
parent_invs_cache[rev.revision_id] = inv
335
repo.add_revision(rev.revision_id, rev)
336
if "verify" in debug.debug_flags:
337
new_unusual_modes = mapping.export_unusual_file_modes(rev)
338
if new_unusual_modes != unusual_modes:
339
raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))
340
objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)
341
for newsha1, newobj, path in objs:
342
assert path is not None
343
if path == "":
344
oldsha1 = o.tree
345
else:
346
(oldmode, oldsha1) = tree_lookup_path(lookup_object, o.tree, path)
347
if oldsha1 != newsha1:
348
raise AssertionError("%r != %r in %s" % (oldsha1, newsha1, path))
349
350
351
def import_git_objects(repo, mapping, object_iter,
352
target_git_object_retriever, heads, pb=None, limit=None):
135
353
"""Import a set of git objects into a bzr repository.
136
354
137
:param repo: Bazaar repository
355
:param repo: Target Bazaar repository
138
356
:param mapping: Mapping to use
139
357
:param object_iter: Iterator over Git objects.
358
:return: Tuple with pack hints and last imported revision id
140
359
"""
360
def lookup_object(sha):
361
try:
362
return object_iter[sha]
363
except KeyError:
364
return target_git_object_retriever[sha]
141
365
# TODO: a more (memory-)efficient implementation of this
142
366
graph = []
143
root_trees = {}
144
revisions = {}
367
checked = set()
368
heads = list(set(heads))
369
parent_invs_cache = lru_cache.LRUSizeCache(compute_size=approx_inv_size,
370
max_size=MAX_INV_CACHE_SIZE)
371
target_git_object_retriever.start_write_group() # FIXME: try/finally
145
372
# Find and convert commit objects
146
for o in object_iter.iterobjects():
373
while heads:
374
if pb is not None:
375
pb.update("finding revisions to fetch", len(graph), None)
376
head = heads.pop()
377
assert isinstance(head, str)
378
try:
379
o = lookup_object(head)
380
except KeyError:
381
continue
147
382
if isinstance(o, Commit):
148
383
rev = mapping.import_commit(o)
149
root_trees[rev.revision_id] = object_iter[o.tree]
150
revisions[rev.revision_id] = rev
151
graph.append((rev.revision_id, rev.parent_ids))
152
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(), "commit", (rev.revision_id, o._tree))
384
if repo.has_revision(rev.revision_id):
385
continue
386
squash_revision(repo, rev)
387
graph.append((o.id, o.parents))
388
target_git_object_retriever._idmap.add_entry(o.id, "commit",
389
(rev.revision_id, o.tree))
390
heads.extend([p for p in o.parents if p not in checked])
391
elif isinstance(o, Tag):
392
if o.object[1] not in checked:
393
heads.append(o.object[1])
394
else:
395
trace.warning("Unable to import head object %r" % o)
396
checked.add(o.id)
397
del checked
153
398
# Order the revisions
154
399
# Create the inventory objects
155
for i, revid in enumerate(topo_sort(graph)):
156
if pb is not None:
157
pb.update("fetching revisions", i, len(graph))
158
root_tree = root_trees[revid]
159
rev = revisions[revid]
160
# We have to do this here, since we have to walk the tree and
161
# we need to make sure to import the blobs / trees with the riht
162
# path; this may involve adding them more than once.
163
inv = Inventory()
164
inv.revision_id = rev.revision_id
165
def lookup_object(sha):
166
if sha in object_iter:
167
return object_iter[sha]
168
return target_git_object_retriever[sha]
169
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
170
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
171
target_git_object_retriever, lookup_object)
172
repo.add_revision(rev.revision_id, rev, inv)
173
174
175
class InterGitNonGitRepository(InterRepository):
176
177
_matching_repo_format = GitFormat()
400
batch_size = 100
401
revision_ids = topo_sort(graph)
402
pack_hints = []
403
if limit is not None:
404
revision_ids = revision_ids[:limit]
405
last_imported = None
406
for offset in range(0, len(revision_ids), batch_size):
407
repo.start_write_group()
408
try:
409
for i, head in enumerate(revision_ids[offset:offset+batch_size]):
410
if pb is not None:
411
pb.update("fetching revisions", offset+i, len(revision_ids))
412
import_git_commit(repo, mapping, head, lookup_object,
413
target_git_object_retriever,
414
parent_invs_cache)
415
last_imported = head
416
except:
417
repo.abort_write_group()
418
raise
419
else:
420
hint = repo.commit_write_group()
421
if hint is not None:
422
pack_hints.extend(hint)
423
target_git_object_retriever.commit_write_group()
424
return pack_hints, last_imported
425
426
427
class InterGitRepository(InterRepository):
428
429
_matching_repo_format = GitRepositoryFormat()
178
430
179
431
@staticmethod
180
432
def _get_repo_format_to_test():
184
436
"""See InterRepository.copy_content."""
185
437
self.fetch(revision_id, pb, find_ghosts=False)
186
438
187
def fetch_objects(self, determine_wants, mapping, pb=None):
439
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
440
mapping=None, fetch_spec=None):
441
self.fetch_refs(revision_id=revision_id, pb=pb,
442
find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)
443
444
445
class InterGitNonGitRepository(InterGitRepository):
446
"""Base InterRepository that copies revisions from a Git into a non-Git
447
repository."""
448
449
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
450
mapping=None, fetch_spec=None):
451
if mapping is None:
452
mapping = self.source.get_mapping()
453
if revision_id is not None:
454
interesting_heads = [revision_id]
455
elif fetch_spec is not None:
456
interesting_heads = fetch_spec.heads
457
else:
458
interesting_heads = None
459
self._refs = {}
460
def determine_wants(refs):
461
self._refs = refs
462
if interesting_heads is None:
463
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
464
else:
465
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
466
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
467
(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)
468
if pack_hint is not None and self.target._format.pack_compresses:
469
self.target.pack(hint=pack_hint)
470
if interesting_heads is not None:
471
present_interesting_heads = self.target.has_revisions(interesting_heads)
472
missing_interesting_heads = set(interesting_heads) - present_interesting_heads
473
if missing_interesting_heads:
474
raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)
475
return self._refs
476
477
478
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
479
def report_git_progress(pb, text):
480
text = text.rstrip("\r\n")
481
g = _GIT_PROGRESS_RE.match(text)
482
if g is not None:
483
(text, pct, current, total) = g.groups()
484
pb.update(text, int(current), int(total))
485
else:
486
pb.update(text, 0, 0)
487
488
489
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
490
"""InterRepository that copies revisions from a remote Git into a non-Git
491
repository."""
492
493
def get_target_heads(self):
494
# FIXME: This should be more efficient
495
all_revs = self.target.all_revision_ids()
496
parent_map = self.target.get_parent_map(all_revs)
497
all_parents = set()
498
map(all_parents.update, parent_map.itervalues())
499
return set(all_revs) - all_parents
500
501
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
188
502
def progress(text):
189
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
190
graph_walker = BzrFetchGraphWalker(self.target, mapping)
503
report_git_progress(pb, text)
504
store = BazaarObjectStore(self.target, mapping)
505
self.target.lock_write()
506
try:
507
heads = self.get_target_heads()
508
graph_walker = store.get_graph_walker(
509
[store._lookup_revision_sha1(head) for head in heads])
510
recorded_wants = []
511
512
def record_determine_wants(heads):
513
wants = determine_wants(heads)
514
recorded_wants.extend(wants)
515
return wants
516
517
create_pb = None
518
if pb is None:
519
create_pb = pb = ui.ui_factory.nested_progress_bar()
520
try:
521
objects_iter = self.source.fetch_objects(
522
record_determine_wants, graph_walker,
523
store.get_raw, progress)
524
return import_git_objects(self.target, mapping,
525
objects_iter, store, recorded_wants, pb, limit)
526
finally:
527
if create_pb:
528
create_pb.finished()
529
finally:
530
self.target.unlock()
531
532
@staticmethod
533
def is_compatible(source, target):
534
"""Be compatible with GitRepository."""
535
# FIXME: Also check target uses VersionedFile
536
return (isinstance(source, RemoteGitRepository) and
537
target.supports_rich_root() and
538
not isinstance(target, GitRepository))
539
540
541
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
542
"""InterRepository that copies revisions from a local Git into a non-Git
543
repository."""
544
545
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
546
"""Fetch objects.
547
"""
548
wants = determine_wants(self.source._git.get_refs())
191
549
create_pb = None
192
550
if pb is None:
193
551
create_pb = pb = ui.ui_factory.nested_progress_bar()
194
target_git_object_retriever = GitObjectConverter(self.target, mapping)
195
552
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
196
553
try:
197
554
self.target.lock_write()
198
555
try:
199
self.target.start_write_group()
200
try:
201
objects_iter = self.source.fetch_objects(determine_wants,
202
graph_walker,
203
target_git_object_retriever.__getitem__,
204
progress)
205
import_git_objects(self.target, mapping, objects_iter,
206
target_git_object_retriever, pb)
207
finally:
208
self.target.commit_write_group()
556
return import_git_objects(self.target, mapping,
557
self.source._git.object_store,
558
target_git_object_retriever, wants, pb, limit)
209
559
finally:
210
560
self.target.unlock()
211
561
finally:
212
562
if create_pb:
213
563
create_pb.finished()
214
564
215
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
216
mapping=None):
217
if mapping is None:
218
mapping = self.source.get_mapping()
219
def determine_wants(heads):
220
if revision_id is None:
221
ret = heads.values()
222
else:
223
ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
224
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
225
return self.fetch_objects(determine_wants, mapping, pb)
226
227
565
@staticmethod
228
566
def is_compatible(source, target):
229
567
"""Be compatible with GitRepository."""
230
568
# FIXME: Also check target uses VersionedFile
231
return (isinstance(source, GitRepository) and
569
return (isinstance(source, LocalGitRepository) and
232
570
target.supports_rich_root() and
233
571
not isinstance(target, GitRepository))
234
572
235
573
236
class InterGitRepository(InterRepository):
237
238
_matching_repo_format = GitFormat()
239
240
@staticmethod
241
def _get_repo_format_to_test():
242
return None
243
244
def copy_content(self, revision_id=None, pb=None):
245
"""See InterRepository.copy_content."""
246
self.fetch(revision_id, pb, find_ghosts=False)
247
248
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
249
mapping=None):
250
if mapping is None:
251
mapping = self.source.get_mapping()
574
class InterGitGitRepository(InterGitRepository):
575
"""InterRepository that copies between Git repositories."""
576
577
def fetch_objects(self, determine_wants, mapping, pb=None):
252
578
def progress(text):
253
info("git: %s", text)
579
trace.note("git: %s", text)
580
graphwalker = self.target._git.get_graph_walker()
581
if (isinstance(self.source, LocalGitRepository) and
582
isinstance(self.target, LocalGitRepository)):
583
return self.source._git.fetch(self.target._git, determine_wants,
584
progress)
585
elif (isinstance(self.source, LocalGitRepository) and
586
isinstance(self.target, RemoteGitRepository)):
587
raise NotImplementedError
588
elif (isinstance(self.source, RemoteGitRepository) and
589
isinstance(self.target, LocalGitRepository)):
590
f, commit = self.target._git.object_store.add_thin_pack()
591
try:
592
refs = self.source._git.fetch_pack(determine_wants,
593
graphwalker, f.write, progress)
594
commit()
595
return refs
596
except:
597
f.close()
598
raise
599
else:
600
raise AssertionError
601
602
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
603
mapping=None, fetch_spec=None, branches=None):
604
if mapping is None:
605
mapping = self.source.get_mapping()
254
606
r = self.target._git
255
if revision_id is None:
256
determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
257
else:
607
if revision_id is not None:
258
608
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
609
elif fetch_spec is not None:
610
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
611
if branches is not None:
612
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
613
elif fetch_spec is None and revision_id is None:
614
determine_wants = r.object_store.determine_wants_all
615
else:
259
616
determine_wants = lambda x: [y for y in args if not y in r.object_store]
617
return self.fetch_objects(determine_wants, mapping)[0]
260
618
261
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
262
f, commit = r.object_store.add_pack()
263
try:
264
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
265
f.close()
266
commit()
267
except:
268
f.close()
269
raise
270
619
271
620
@staticmethod
272
621
def is_compatible(source, target):
273
622
"""Be compatible with GitRepository."""
274
return (isinstance(source, GitRepository) and
623
return (isinstance(source, GitRepository) and
275
624
isinstance(target, GitRepository))
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0