To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.200.769
- compare with another revision
- download diff
- download tarball
- view history from revision 0.200.769
- Committer: Jelmer Vernooij
- Date: 2010-03-28 21:52:29 UTC
- mto: (0.312.1 master) (6883.23.1 bundle-git)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@samba.org-20100328215229-4u280rwxg9w87pa0
Cope with open_branch() actually checking whether there is a branch present.
- files added:
- HACKING
- files removed:
- foreign
- files renamed:
- converter.py => object_store.py
- tests/test_ids.py => tests/test_mapping.py
- files modified:
- Makefile
added
removed
fetch.py
1
# Copyright (C) 2008 Canonical Ltd
1
# Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
from bzrlib import osutils, ui, urlutils
18
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
19
from bzrlib.inventory import Inventory
20
from bzrlib.repository import InterRepository
21
from bzrlib.trace import info
22
from bzrlib.tsort import topo_sort
23
17
from dulwich.objects import (
18
Commit,
19
Tag,
20
S_ISGITLINK,
21
)
22
from dulwich.object_store import (
23
tree_lookup_path,
24
)
25
import re
26
import stat
27
28
from bzrlib import (
29
debug,
30
lru_cache,
31
osutils,
32
trace,
33
ui,
34
urlutils,
35
)
36
from bzrlib.errors import (
37
BzrError,
38
NoSuchId,
39
)
40
from bzrlib.inventory import (
41
Inventory,
42
InventoryDirectory,
43
InventoryFile,
44
InventoryLink,
45
TreeReference,
46
)
47
from bzrlib.repository import (
48
InterRepository,
49
)
50
from bzrlib.revision import (
51
NULL_REVISION,
52
)
53
from bzrlib.tsort import (
54
topo_sort,
55
)
56
from bzrlib.versionedfile import (
57
FulltextContentFactory,
58
)
59
60
from bzrlib.plugins.git.mapping import (
61
DEFAULT_FILE_MODE,
62
inventory_to_tree_and_blobs,
63
mode_is_executable,
64
squash_revision,
65
warn_unusual_mode,
66
)
67
from bzrlib.plugins.git.object_store import (
68
BazaarObjectStore,
69
)
70
from bzrlib.plugins.git.remote import (
71
RemoteGitRepository,
72
)
24
73
from bzrlib.plugins.git.repository import (
25
LocalGitRepository,
26
GitRepository,
27
GitFormat,
28
)
29
from bzrlib.plugins.git.converter import GitObjectConverter
30
from bzrlib.plugins.git.remote import RemoteGitRepository
31
32
import dulwich as git
33
from dulwich.client import SimpleFetchGraphWalker
34
from dulwich.objects import Commit
35
36
from cStringIO import StringIO
37
38
39
class BzrFetchGraphWalker(object):
40
"""GraphWalker implementation that uses a Bazaar repository."""
41
42
def __init__(self, repository, mapping):
43
self.repository = repository
44
self.mapping = mapping
45
self.done = set()
46
self.heads = set(repository.all_revision_ids())
47
self.parents = {}
48
49
def __iter__(self):
50
return iter(self.next, None)
51
52
def ack(self, sha):
53
revid = self.mapping.revision_id_foreign_to_bzr(sha)
54
self.remove(revid)
55
56
def remove(self, revid):
57
self.done.add(revid)
58
if revid in self.heads:
59
self.heads.remove(revid)
60
if revid in self.parents:
61
for p in self.parents[revid]:
62
self.remove(p)
63
64
def next(self):
65
while self.heads:
66
ret = self.heads.pop()
67
ps = self.repository.get_parent_map([ret])[ret]
68
self.parents[ret] = ps
69
self.heads.update([p for p in ps if not p in self.done])
70
try:
71
self.done.add(ret)
72
return self.mapping.revision_id_bzr_to_foreign(ret)[0]
73
except InvalidRevisionId:
74
pass
75
return None
76
77
78
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, gitmap, executable):
74
GitRepository,
75
GitRepositoryFormat,
76
LocalGitRepository,
77
)
78
79
80
MAX_INV_CACHE_SIZE = 50 * 1024 * 1024
81
82
83
def import_git_blob(texts, mapping, path, hexsha, base_inv, base_inv_shamap,
84
base_ie, parent_id, revision_id, parent_invs, lookup_object,
85
executable, symlink):
79
86
"""Import a git blob object into a bzr repository.
80
87
81
:param repo: bzr repository
88
:param texts: VersionedFiles to add to
82
89
:param path: Path in the tree
83
90
:param blob: A git blob
91
:return: Inventory delta for this file
84
92
"""
85
93
file_id = mapping.generate_file_id(path)
86
text_revision = inv.revision_id
87
repo.texts.add_lines((file_id, text_revision),
88
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
89
osutils.split_lines(blob.data))
90
ie = inv.add_path(path, "file", file_id)
91
ie.revision = text_revision
92
ie.text_size = len(blob.data)
93
ie.text_sha1 = osutils.sha_string(blob.data)
94
if symlink:
95
cls = InventoryLink
96
else:
97
cls = InventoryFile
98
# We just have to hope this is indeed utf-8:
99
ie = cls(file_id, urlutils.basename(path).decode("utf-8"), parent_id)
94
100
ie.executable = executable
95
gitmap._idmap.add_entry(blob.sha().hexdigest(), "blob", (ie.file_id, ie.revision))
96
97
98
def import_git_tree(repo, mapping, path, tree, inv, parent_invs,
99
gitmap, lookup_object):
101
# See if this has changed at all
102
if base_ie is None:
103
base_sha = None
104
else:
105
try:
106
base_sha = base_inv_shamap.lookup_blob(file_id,
107
revision_hint=base_ie.revision)
108
except KeyError:
109
base_sha = None
110
else:
111
if (base_sha == hexsha and base_ie.executable == ie.executable
112
and base_ie.kind == ie.kind):
113
# If nothing has changed since the base revision, we're done
114
return [], []
115
if base_sha == hexsha and base_ie.kind == ie.kind:
116
ie.text_size = base_ie.text_size
117
ie.text_sha1 = base_ie.text_sha1
118
ie.symlink_target = base_ie.symlink_target
119
if ie.executable == base_ie.executable:
120
ie.revision = base_ie.revision
121
else:
122
blob = lookup_object(hexsha)
123
else:
124
blob = lookup_object(hexsha)
125
if ie.kind == "symlink":
126
ie.revision = None
127
ie.symlink_target = blob.data
128
ie.text_size = None
129
ie.text_sha1 = None
130
else:
131
ie.text_size = len(blob.data)
132
ie.text_sha1 = osutils.sha_string(blob.data)
133
# Check what revision we should store
134
parent_keys = []
135
for pinv in parent_invs:
136
if pinv.revision_id == base_inv.revision_id:
137
pie = base_ie
138
if pie is None:
139
continue
140
else:
141
try:
142
pie = pinv[file_id]
143
except NoSuchId:
144
continue
145
if pie.text_sha1 == ie.text_sha1 and pie.executable == ie.executable and pie.symlink_target == ie.symlink_target:
146
# found a revision in one of the parents to use
147
ie.revision = pie.revision
148
break
149
parent_keys.append((file_id, pie.revision))
150
if ie.revision is None:
151
# Need to store a new revision
152
ie.revision = revision_id
153
assert file_id is not None
154
assert ie.revision is not None
155
if ie.kind == 'symlink':
156
data = ''
157
else:
158
data = blob.data
159
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), tuple(parent_keys), ie.text_sha1, data)])
160
shamap = { ie.file_id: hexsha }
161
invdelta = []
162
if base_ie is not None:
163
old_path = base_inv.id2path(file_id)
164
if base_ie.kind == "directory":
165
invdelta.extend(remove_disappeared_children(old_path, base_ie.children, []))
166
else:
167
old_path = None
168
invdelta.append((old_path, path, file_id, ie))
169
return (invdelta, shamap)
170
171
172
class SubmodulesRequireSubtrees(BzrError):
173
_fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
174
internal = False
175
176
177
def import_git_submodule(texts, mapping, path, hexsha, base_inv, base_ie,
178
parent_id, revision_id, parent_invs, lookup_object):
179
file_id = mapping.generate_file_id(path)
180
ie = TreeReference(file_id, urlutils.basename(path.decode("utf-8")),
181
parent_id)
182
ie.revision = revision_id
183
if base_ie is None:
184
oldpath = None
185
else:
186
oldpath = path
187
if (base_ie.kind == ie.kind and
188
base_ie.reference_revision == ie.reference_revision):
189
ie.revision = base_ie.revision
190
ie.reference_revision = mapping.revision_id_foreign_to_bzr(hexsha)
191
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
192
invdelta = [(oldpath, path, file_id, ie)]
193
return invdelta, {}, {}
194
195
196
def remove_disappeared_children(path, base_children, existing_children):
197
ret = []
198
deletable = [(osutils.pathjoin(path, k), v) for k,v in base_children.iteritems() if k not in existing_children]
199
while deletable:
200
(path, ie) = deletable.pop()
201
ret.append((path, None, ie.file_id, None))
202
if ie.kind == "directory":
203
for name, child_ie in ie.children.iteritems():
204
deletable.append((osutils.pathjoin(path, name), child_ie))
205
return ret
206
207
208
def import_git_tree(texts, mapping, path, hexsha, base_inv, base_inv_shamap,
209
base_ie, parent_id, revision_id, parent_invs, lookup_object,
210
allow_submodules=False):
100
211
"""Import a git tree object into a bzr repository.
101
212
102
:param repo: A Bzr repository object
213
:param texts: VersionedFiles object to add to
103
214
:param path: Path in the tree
104
215
:param tree: A git tree object
105
:param inv: Inventory object
216
:param base_inv: Base inventory against which to return inventory delta
217
:return: Inventory delta for this subtree
106
218
"""
219
invdelta = []
107
220
file_id = mapping.generate_file_id(path)
108
text_revision = inv.revision_id
109
repo.texts.add_lines((file_id, text_revision),
110
[(file_id, p[file_id].revision) for p in parent_invs if file_id in p],
111
[])
112
ie = inv.add_path(path, "directory", file_id)
113
ie.revision = text_revision
114
gitmap._idmap.add_entry(tree.sha().hexdigest(), "tree", (file_id, text_revision))
115
for mode, name, hexsha in tree.entries():
116
entry_kind = (mode & 0700000) / 0100000
221
# We just have to hope this is indeed utf-8:
222
ie = InventoryDirectory(file_id, urlutils.basename(path.decode("utf-8")),
223
parent_id)
224
if base_ie is None:
225
# Newly appeared here
226
ie.revision = revision_id
227
texts.insert_record_stream([FulltextContentFactory((file_id, ie.revision), (), None, "")])
228
invdelta.append((None, path, file_id, ie))
229
else:
230
# See if this has changed at all
231
try:
232
base_sha = base_inv_shamap.lookup_tree(file_id)
233
except (KeyError, NotImplementedError):
234
pass
235
else:
236
if base_sha == hexsha:
237
# If nothing has changed since the base revision, we're done
238
return [], {}, []
239
if base_ie.kind != "directory":
240
ie.revision = revision_id
241
texts.insert_record_stream([FulltextContentFactory((ie.file_id, ie.revision), (), None, "")])
242
invdelta.append((base_inv.id2path(ie.file_id), path, ie.file_id, ie))
243
if base_ie is not None and base_ie.kind == "directory":
244
base_children = base_ie.children
245
else:
246
base_children = {}
247
# Remember for next time
248
existing_children = set()
249
child_modes = {}
250
shamap = {}
251
tree = lookup_object(hexsha)
252
for mode, name, child_hexsha in tree.entries():
117
253
basename = name.decode("utf-8")
118
if path == "":
119
child_path = name
120
else:
121
child_path = urlutils.join(path, name)
122
if entry_kind == 0:
123
tree = lookup_object(hexsha)
124
import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, gitmap, lookup_object)
125
elif entry_kind == 1:
126
blob = lookup_object(hexsha)
127
fs_mode = mode & 0777
128
import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, gitmap, bool(fs_mode & 0111))
129
else:
130
raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
131
132
133
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever,
134
pb=None):
254
existing_children.add(basename)
255
child_path = osutils.pathjoin(path, name)
256
if stat.S_ISDIR(mode):
257
subinvdelta, grandchildmodes, subshamap = import_git_tree(
258
texts, mapping, child_path, child_hexsha, base_inv,
259
base_inv_shamap, base_children.get(basename), file_id,
260
revision_id, parent_invs, lookup_object,
261
allow_submodules=allow_submodules)
262
elif S_ISGITLINK(mode): # submodule
263
if not allow_submodules:
264
raise SubmodulesRequireSubtrees()
265
subinvdelta, grandchildmodes, subshamap = import_git_submodule(
266
texts, mapping, child_path, child_hexsha, base_inv, base_children.get(basename),
267
file_id, revision_id, parent_invs, lookup_object)
268
else:
269
subinvdelta, subshamap = import_git_blob(texts, mapping,
270
child_path, child_hexsha, base_inv, base_inv_shamap,
271
base_children.get(basename), file_id,
272
revision_id, parent_invs, lookup_object,
273
mode_is_executable(mode), stat.S_ISLNK(mode))
274
grandchildmodes = {}
275
child_modes.update(grandchildmodes)
276
invdelta.extend(subinvdelta)
277
shamap.update(subshamap)
278
if mode not in (stat.S_IFDIR, DEFAULT_FILE_MODE,
279
stat.S_IFLNK, DEFAULT_FILE_MODE|0111):
280
child_modes[child_path] = mode
281
# Remove any children that have disappeared
282
if base_ie is not None and base_ie.kind == "directory":
283
invdelta.extend(remove_disappeared_children(base_inv.id2path(file_id),
284
base_children, existing_children))
285
shamap[file_id] = hexsha
286
return invdelta, child_modes, shamap
287
288
289
def approx_inv_size(inv):
290
# Very rough estimate, 1k per inventory entry
291
return len(inv) * 1024
292
293
294
def import_git_commit(repo, mapping, head, lookup_object,
295
target_git_object_retriever, parent_invs_cache):
296
o = lookup_object(head)
297
rev = mapping.import_commit(o)
298
# We have to do this here, since we have to walk the tree and
299
# we need to make sure to import the blobs / trees with the right
300
# path; this may involve adding them more than once.
301
parent_invs = []
302
for parent_id in rev.parent_ids:
303
try:
304
parent_invs.append(parent_invs_cache[parent_id])
305
except KeyError:
306
parent_inv = repo.get_inventory(parent_id)
307
parent_invs.append(parent_inv)
308
parent_invs_cache[parent_id] = parent_inv
309
if parent_invs == []:
310
base_inv = Inventory(root_id=None)
311
base_ie = None
312
base_inv_shamap = None # Should never be accessed
313
else:
314
base_inv = parent_invs[0]
315
base_ie = base_inv.root
316
base_inv_shamap = target_git_object_retriever._idmap.get_inventory_sha_map(base_inv.revision_id)
317
inv_delta, unusual_modes, shamap = import_git_tree(repo.texts,
318
mapping, "", o.tree, base_inv, base_inv_shamap, base_ie, None,
319
rev.revision_id, parent_invs, lookup_object,
320
allow_submodules=getattr(repo._format, "supports_tree_reference", False))
321
entries = []
322
for (oldpath, newpath, fileid, new_ie) in inv_delta:
323
if newpath is None:
324
entries.append((fileid, None, None, None))
325
else:
326
if new_ie.kind in ("file", "symlink"):
327
entries.append((fileid, "blob", shamap[fileid], new_ie.revision))
328
elif new_ie.kind == "directory":
329
entries.append((fileid, "tree", shamap[fileid], rev.revision_id))
330
else:
331
raise AssertionError
332
target_git_object_retriever._idmap.add_entries(rev.revision_id,
333
rev.parent_ids, head, o.tree, entries)
334
if unusual_modes != {}:
335
for path, mode in unusual_modes.iteritems():
336
warn_unusual_mode(rev.foreign_revid, path, mode)
337
mapping.import_unusual_file_modes(rev, unusual_modes)
338
try:
339
basis_id = rev.parent_ids[0]
340
except IndexError:
341
basis_id = NULL_REVISION
342
base_inv = None
343
rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
344
inv_delta, rev.revision_id, rev.parent_ids,
345
base_inv)
346
parent_invs_cache[rev.revision_id] = inv
347
repo.add_revision(rev.revision_id, rev)
348
if "verify" in debug.debug_flags:
349
new_unusual_modes = mapping.export_unusual_file_modes(rev)
350
if new_unusual_modes != unusual_modes:
351
raise AssertionError("unusual modes don't match: %r != %r" % (unusual_modes, new_unusual_modes))
352
objs = inventory_to_tree_and_blobs(inv, repo.texts, mapping, unusual_modes)
353
for newsha1, newobj, path in objs:
354
assert path is not None
355
if path == "":
356
oldsha1 = o.tree
357
else:
358
(oldmode, oldsha1) = tree_lookup_path(lookup_object, o.tree, path)
359
if oldsha1 != newsha1:
360
raise AssertionError("%r != %r in %s" % (oldsha1, newsha1, path))
361
362
363
def import_git_objects(repo, mapping, object_iter,
364
target_git_object_retriever, heads, pb=None, limit=None):
135
365
"""Import a set of git objects into a bzr repository.
136
366
137
:param repo: Bazaar repository
367
:param repo: Target Bazaar repository
138
368
:param mapping: Mapping to use
139
369
:param object_iter: Iterator over Git objects.
370
:return: Tuple with pack hints and last imported revision id
140
371
"""
372
def lookup_object(sha):
373
try:
374
return object_iter[sha]
375
except KeyError:
376
return target_git_object_retriever[sha]
141
377
# TODO: a more (memory-)efficient implementation of this
142
378
graph = []
143
root_trees = {}
144
revisions = {}
379
checked = set()
380
heads = list(set(heads))
381
parent_invs_cache = lru_cache.LRUSizeCache(compute_size=approx_inv_size,
382
max_size=MAX_INV_CACHE_SIZE)
383
target_git_object_retriever.start_write_group() # FIXME: try/finally
145
384
# Find and convert commit objects
146
for o in object_iter.iterobjects():
385
while heads:
386
if pb is not None:
387
pb.update("finding revisions to fetch", len(graph), None)
388
head = heads.pop()
389
assert isinstance(head, str)
390
try:
391
o = lookup_object(head)
392
except KeyError:
393
continue
147
394
if isinstance(o, Commit):
148
395
rev = mapping.import_commit(o)
149
root_trees[rev.revision_id] = object_iter[o.tree]
150
revisions[rev.revision_id] = rev
151
graph.append((rev.revision_id, rev.parent_ids))
152
target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(), "commit", (rev.revision_id, o._tree))
396
if repo.has_revision(rev.revision_id):
397
continue
398
squash_revision(repo, rev)
399
graph.append((o.id, o.parents))
400
heads.extend([p for p in o.parents if p not in checked])
401
elif isinstance(o, Tag):
402
if o.object[1] not in checked:
403
heads.append(o.object[1])
404
else:
405
trace.warning("Unable to import head object %r" % o)
406
checked.add(o.id)
407
del checked
153
408
# Order the revisions
154
409
# Create the inventory objects
155
for i, revid in enumerate(topo_sort(graph)):
156
if pb is not None:
157
pb.update("fetching revisions", i, len(graph))
158
root_tree = root_trees[revid]
159
rev = revisions[revid]
160
# We have to do this here, since we have to walk the tree and
161
# we need to make sure to import the blobs / trees with the riht
162
# path; this may involve adding them more than once.
163
inv = Inventory()
164
inv.revision_id = rev.revision_id
165
def lookup_object(sha):
166
if sha in object_iter:
167
return object_iter[sha]
168
return target_git_object_retriever[sha]
169
parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
170
import_git_tree(repo, mapping, "", root_tree, inv, parent_invs,
171
target_git_object_retriever, lookup_object)
172
repo.add_revision(rev.revision_id, rev, inv)
173
174
175
class InterGitNonGitRepository(InterRepository):
176
177
_matching_repo_format = GitFormat()
410
batch_size = 100
411
revision_ids = topo_sort(graph)
412
pack_hints = []
413
if limit is not None:
414
revision_ids = revision_ids[:limit]
415
last_imported = None
416
for offset in range(0, len(revision_ids), batch_size):
417
repo.start_write_group()
418
try:
419
for i, head in enumerate(revision_ids[offset:offset+batch_size]):
420
if pb is not None:
421
pb.update("fetching revisions", offset+i, len(revision_ids))
422
import_git_commit(repo, mapping, head, lookup_object,
423
target_git_object_retriever,
424
parent_invs_cache)
425
last_imported = head
426
except:
427
repo.abort_write_group()
428
raise
429
else:
430
hint = repo.commit_write_group()
431
if hint is not None:
432
pack_hints.extend(hint)
433
target_git_object_retriever.commit_write_group()
434
return pack_hints, last_imported
435
436
437
class InterGitRepository(InterRepository):
438
439
_matching_repo_format = GitRepositoryFormat()
178
440
179
441
@staticmethod
180
442
def _get_repo_format_to_test():
184
446
"""See InterRepository.copy_content."""
185
447
self.fetch(revision_id, pb, find_ghosts=False)
186
448
187
def fetch_objects(self, determine_wants, mapping, pb=None):
449
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
450
mapping=None, fetch_spec=None):
451
self.fetch_refs(revision_id=revision_id, pb=pb,
452
find_ghosts=find_ghosts, mapping=mapping, fetch_spec=fetch_spec)
453
454
455
class InterGitNonGitRepository(InterGitRepository):
456
"""Base InterRepository that copies revisions from a Git into a non-Git
457
repository."""
458
459
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
460
mapping=None, fetch_spec=None):
461
if mapping is None:
462
mapping = self.source.get_mapping()
463
if revision_id is not None:
464
interesting_heads = [revision_id]
465
elif fetch_spec is not None:
466
interesting_heads = fetch_spec.heads
467
else:
468
interesting_heads = None
469
self._refs = {}
470
def determine_wants(refs):
471
self._refs = refs
472
if interesting_heads is None:
473
ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
474
else:
475
ret = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
476
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
477
(pack_hint, _) = self.fetch_objects(determine_wants, mapping, pb)
478
if pack_hint is not None and self.target._format.pack_compresses:
479
self.target.pack(hint=pack_hint)
480
if interesting_heads is not None:
481
present_interesting_heads = self.target.has_revisions(interesting_heads)
482
missing_interesting_heads = set(interesting_heads) - present_interesting_heads
483
if missing_interesting_heads:
484
raise AssertionError("Missing interesting heads: %r" % missing_interesting_heads)
485
return self._refs
486
487
488
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
489
def report_git_progress(pb, text):
490
text = text.rstrip("\r\n")
491
g = _GIT_PROGRESS_RE.match(text)
492
if g is not None:
493
(text, pct, current, total) = g.groups()
494
pb.update(text, int(current), int(total))
495
else:
496
pb.update(text, 0, 0)
497
498
499
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
500
"""InterRepository that copies revisions from a remote Git into a non-Git
501
repository."""
502
503
def get_target_heads(self):
504
# FIXME: This should be more efficient
505
all_revs = self.target.all_revision_ids()
506
parent_map = self.target.get_parent_map(all_revs)
507
all_parents = set()
508
map(all_parents.update, parent_map.itervalues())
509
return set(all_revs) - all_parents
510
511
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
188
512
def progress(text):
189
pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
190
graph_walker = BzrFetchGraphWalker(self.target, mapping)
513
report_git_progress(pb, text)
514
store = BazaarObjectStore(self.target, mapping)
515
self.target.lock_write()
516
try:
517
heads = self.get_target_heads()
518
graph_walker = store.get_graph_walker(
519
[store._lookup_revision_sha1(head) for head in heads])
520
recorded_wants = []
521
522
def record_determine_wants(heads):
523
wants = determine_wants(heads)
524
recorded_wants.extend(wants)
525
return wants
526
527
create_pb = None
528
if pb is None:
529
create_pb = pb = ui.ui_factory.nested_progress_bar()
530
try:
531
objects_iter = self.source.fetch_objects(
532
record_determine_wants, graph_walker,
533
store.get_raw, progress)
534
return import_git_objects(self.target, mapping,
535
objects_iter, store, recorded_wants, pb, limit)
536
finally:
537
if create_pb:
538
create_pb.finished()
539
finally:
540
self.target.unlock()
541
542
@staticmethod
543
def is_compatible(source, target):
544
"""Be compatible with GitRepository."""
545
# FIXME: Also check target uses VersionedFile
546
return (isinstance(source, RemoteGitRepository) and
547
target.supports_rich_root() and
548
not isinstance(target, GitRepository))
549
550
551
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
552
"""InterRepository that copies revisions from a local Git into a non-Git
553
repository."""
554
555
def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
556
"""Fetch objects.
557
"""
558
wants = determine_wants(self.source._git.get_refs())
191
559
create_pb = None
192
560
if pb is None:
193
561
create_pb = pb = ui.ui_factory.nested_progress_bar()
194
target_git_object_retriever = GitObjectConverter(self.target, mapping)
195
562
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
196
563
try:
197
564
self.target.lock_write()
198
565
try:
199
self.target.start_write_group()
200
try:
201
objects_iter = self.source.fetch_objects(determine_wants,
202
graph_walker,
203
target_git_object_retriever.__getitem__,
204
progress)
205
import_git_objects(self.target, mapping, objects_iter,
206
target_git_object_retriever, pb)
207
finally:
208
self.target.commit_write_group()
566
return import_git_objects(self.target, mapping,
567
self.source._git.object_store,
568
target_git_object_retriever, wants, pb, limit)
209
569
finally:
210
570
self.target.unlock()
211
571
finally:
212
572
if create_pb:
213
573
create_pb.finished()
214
574
215
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
216
mapping=None):
217
if mapping is None:
218
mapping = self.source.get_mapping()
219
def determine_wants(heads):
220
if revision_id is None:
221
ret = heads.values()
222
else:
223
ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
224
return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
225
return self.fetch_objects(determine_wants, mapping, pb)
226
227
575
@staticmethod
228
576
def is_compatible(source, target):
229
577
"""Be compatible with GitRepository."""
230
578
# FIXME: Also check target uses VersionedFile
231
return (isinstance(source, GitRepository) and
579
return (isinstance(source, LocalGitRepository) and
232
580
target.supports_rich_root() and
233
581
not isinstance(target, GitRepository))
234
582
235
583
236
class InterGitRepository(InterRepository):
237
238
_matching_repo_format = GitFormat()
239
240
@staticmethod
241
def _get_repo_format_to_test():
242
return None
243
244
def copy_content(self, revision_id=None, pb=None):
245
"""See InterRepository.copy_content."""
246
self.fetch(revision_id, pb, find_ghosts=False)
247
248
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
249
mapping=None):
250
if mapping is None:
251
mapping = self.source.get_mapping()
584
class InterGitGitRepository(InterGitRepository):
585
"""InterRepository that copies between Git repositories."""
586
587
def fetch_objects(self, determine_wants, mapping, pb=None):
252
588
def progress(text):
253
info("git: %s", text)
589
trace.note("git: %s", text)
590
graphwalker = self.target._git.get_graph_walker()
591
if (isinstance(self.source, LocalGitRepository) and
592
isinstance(self.target, LocalGitRepository)):
593
return self.source._git.fetch(self.target._git, determine_wants,
594
progress)
595
elif (isinstance(self.source, LocalGitRepository) and
596
isinstance(self.target, RemoteGitRepository)):
597
raise NotImplementedError
598
elif (isinstance(self.source, RemoteGitRepository) and
599
isinstance(self.target, LocalGitRepository)):
600
f, commit = self.target._git.object_store.add_thin_pack()
601
try:
602
refs = self.source._git.fetch_pack(determine_wants,
603
graphwalker, f.write, progress)
604
commit()
605
return refs
606
except:
607
f.close()
608
raise
609
else:
610
raise AssertionError
611
612
def fetch_refs(self, revision_id=None, pb=None, find_ghosts=False,
613
mapping=None, fetch_spec=None, branches=None):
614
if mapping is None:
615
mapping = self.source.get_mapping()
254
616
r = self.target._git
255
if revision_id is None:
256
determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
257
else:
617
if revision_id is not None:
258
618
args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
619
elif fetch_spec is not None:
620
args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
621
if branches is not None:
622
determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
623
elif fetch_spec is None and revision_id is None:
624
determine_wants = r.object_store.determine_wants_all
625
else:
259
626
determine_wants = lambda x: [y for y in args if not y in r.object_store]
627
return self.fetch_objects(determine_wants, mapping)[0]
260
628
261
graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
262
f, commit = r.object_store.add_pack()
263
try:
264
self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
265
f.close()
266
commit()
267
except:
268
f.close()
269
raise
270
629
271
630
@staticmethod
272
631
def is_compatible(source, target):
273
632
"""Be compatible with GitRepository."""
274
return (isinstance(source, GitRepository) and
633
return (isinstance(source, GitRepository) and
275
634
isinstance(target, GitRepository))
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0