147
140
raise NoSuchRevision(revid, self.source)
148
141
git_shas.append(git_sha)
142
walker = Walker(self.source_store,
143
include=git_shas, exclude=[
153
144
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
154
145
if sha != ZERO_SHA])
155
146
missing_revids = set()
156
147
for entry in walker:
157
for (kind, type_data) in self.source_store.lookup_git_sha(
148
for (kind, type_data) in self.source_store.lookup_git_sha(entry.commit.id):
159
149
if kind == "commit":
160
150
missing_revids.add(type_data[0])
161
151
return self.source.revision_ids_to_search_result(missing_revids)
218
208
stop_revids.append(revid)
220
210
graph = self.source.get_graph()
221
with ui.ui_factory.nested_progress_bar() as pb:
211
pb = ui.ui_factory.nested_progress_bar()
222
213
while stop_revids:
223
214
new_stop_revids = []
224
215
for revid in stop_revids:
225
216
sha1 = revid_sha_map.get(revid)
226
if (revid not in missing and
227
self._revision_needs_fetching(sha1, revid)):
217
if (not revid in missing and
218
self._revision_needs_fetching(sha1, revid)):
228
219
missing.add(revid)
229
220
new_stop_revids.append(revid)
230
221
stop_revids = set()
231
222
parent_map = graph.get_parent_map(new_stop_revids)
232
for parent_revids in viewvalues(parent_map):
223
for parent_revids in parent_map.itervalues():
233
224
stop_revids.update(parent_revids)
234
225
pb.update("determining revisions to fetch", len(missing))
235
228
return graph.iter_topo_order(missing)
237
230
def _get_target_bzr_refs(self):
264
256
def fetch_refs(self, update_refs, lossy, overwrite=False):
265
257
self._warn_slow()
267
258
with self.source_store.lock_read():
268
259
old_refs = self._get_target_bzr_refs()
269
260
new_refs = update_refs(old_refs)
270
261
revidmap = self.fetch_objects(
271
[(git_sha, bzr_revid)
272
for (git_sha, bzr_revid) in new_refs.values()
273
if git_sha is None or not git_sha.startswith(SYMREF)],
275
for name, (gitid, revid) in viewitems(new_refs):
262
[(git_sha, bzr_revid) for (git_sha, bzr_revid) in new_refs.values() if git_sha is None or not git_sha.startswith(SYMREF)], lossy=lossy)
263
for name, (gitid, revid) in new_refs.iteritems():
276
264
if gitid is None:
278
266
gitid = revidmap[revid][0]
280
268
gitid = self.source_store._lookup_revision_sha1(revid)
281
269
if gitid.startswith(SYMREF):
282
self.target_refs.set_symbolic_ref(
283
name, gitid[len(SYMREF):])
270
self.target_refs.set_symbolic_ref(name, gitid[len(SYMREF):])
286
273
old_git_id = old_refs[name][0]
288
275
self.target_refs.add_if_new(name, gitid)
290
277
self.target_refs.set_if_equals(name, old_git_id, gitid)
291
result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
292
return revidmap, old_refs, result_refs
278
return revidmap, old_refs, new_refs
294
280
def fetch_objects(self, revs, lossy, limit=None):
295
281
if not lossy and not self.mapping.roundtripping:
296
282
for git_sha, bzr_revid in revs:
297
if (bzr_revid is not None and
298
needs_roundtripping(self.source, bzr_revid)):
283
if bzr_revid is not None and needs_roundtripping(self.source, bzr_revid):
299
284
raise NoPushSupport(self.source, self.target, self.mapping,
301
286
with self.source_store.lock_read():
302
287
todo = list(self.missing_revisions(revs))[:limit]
304
with ui.ui_factory.nested_progress_bar() as pb:
289
pb = ui.ui_factory.nested_progress_bar()
305
291
object_generator = MissingObjectsIterator(
306
292
self.source_store, self.source, pb)
307
293
for (old_revid, git_sha) in object_generator.import_revisions(
310
new_revid = self.mapping.revision_id_foreign_to_bzr(
296
new_revid = self.mapping.revision_id_foreign_to_bzr(git_sha)
313
298
new_revid = old_revid
319
304
revidmap[old_revid] = (git_sha, new_revid)
320
305
self.target_store.add_objects(object_generator)
323
310
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
324
fetch_spec=None, mapped_refs=None, lossy=False):
311
fetch_spec=None, mapped_refs=None):
325
312
if mapped_refs is not None:
326
313
stop_revisions = mapped_refs
327
314
elif revision_id is not None:
331
318
if recipe[0] in ("search", "proxy-search"):
332
319
stop_revisions = [(None, revid) for revid in recipe[1]]
334
raise AssertionError(
335
"Unsupported search result type %s" % recipe[0])
321
raise AssertionError("Unsupported search result type %s" % recipe[0])
337
stop_revisions = [(None, revid)
338
for revid in self.source.all_revision_ids()]
323
stop_revisions = [(None, revid) for revid in self.source.all_revision_ids()]
339
324
self._warn_slow()
341
revidmap = self.fetch_objects(stop_revisions, lossy=lossy)
326
self.fetch_objects(stop_revisions, lossy=False)
342
327
except NoPushSupport:
343
328
raise NoRoundtrippingSupport(self.source, self.target)
344
return FetchResult(revidmap)
347
331
def is_compatible(source, target):
358
342
raise NoPushSupport(self.source, self.target, self.mapping)
359
343
unpeel_map = UnpeelMap.from_repository(self.source)
362
def git_update_refs(old_refs):
345
def determine_wants(old_refs):
365
k: (v, None) for (k, v) in viewitems(old_refs)}
366
new_refs = update_refs(self.old_refs)
367
for name, (gitid, revid) in viewitems(new_refs):
347
self.old_refs = dict([(k, (v, None)) for (k, v) in old_refs.iteritems()])
348
self.new_refs = update_refs(self.old_refs)
349
for name, (gitid, revid) in self.new_refs.iteritems():
368
350
if gitid is None:
369
351
git_sha = self.source_store._lookup_revision_sha1(revid)
370
gitid = unpeel_map.re_unpeel_tag(
371
git_sha, old_refs.get(name))
352
gitid = unpeel_map.re_unpeel_tag(git_sha, old_refs.get(name))
372
353
if not overwrite:
373
if remote_divergence(
374
old_refs.get(name), gitid, self.source_store):
354
if remote_divergence(old_refs.get(name), gitid, self.source_store):
375
355
raise DivergedBranches(self.source, self.target)
376
356
ret[name] = gitid
378
358
self._warn_slow()
379
359
with self.source_store.lock_read():
380
new_refs = self.target.send_pack(
381
git_update_refs, self.source_store.generate_lossy_pack_data)
360
new_refs = self.target.send_pack(determine_wants,
361
self.source_store.generate_lossy_pack_data)
382
362
# FIXME: revidmap?
383
return revidmap, self.old_refs, new_refs
363
return revidmap, self.old_refs, self.new_refs
386
366
def is_compatible(source, target):
410
390
def _target_has_shas(self, shas):
411
391
raise NotImplementedError(self._target_has_shas)
413
def get_determine_wants_heads(self, wants, include_tags=False, tag_selector=None):
393
def get_determine_wants_heads(self, wants, include_tags=False):
414
394
wants = set(wants)
416
395
def determine_wants(refs):
418
for k, v in viewitems(refs):
419
if k.endswith(ANNOTATED_TAG_SUFFIX):
420
unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
421
potential = set([unpeel_lookup.get(w, w) for w in wants])
396
potential = set(wants)
423
for k, sha in viewitems(refs):
424
if k.endswith(ANNOTATED_TAG_SUFFIX):
427
tag_name = ref_to_tag_name(k)
430
if tag_selector and not tag_selector(tag_name):
398
for k, unpeeled in refs.iteritems():
399
if k.endswith("^{}"):
403
if unpeeled == ZERO_SHA:
405
potential.add(unpeeled)
435
406
return list(potential - self._target_has_shas(potential))
436
407
return determine_wants
447
418
self.fetch(revision_id, find_ghosts=False)
449
420
def search_missing_revision_ids(self,
450
find_ghosts=True, revision_ids=None,
451
if_present_ids=None, limit=None):
421
find_ghosts=True, revision_ids=None, if_present_ids=None,
452
423
if limit is not None:
453
424
raise FetchLimitUnsupported(self)
454
425
if revision_ids is None and if_present_ids is None:
490
461
def determine_wants_all(self, refs):
491
462
potential = set()
492
for k, v in viewitems(refs):
463
for k, v in refs.iteritems():
493
464
# For non-git target repositories, only worry about peeled
494
465
if v == ZERO_SHA:
496
467
potential.add(self.source.controldir.get_peeled(k) or v)
497
468
return list(potential - self._target_has_shas(potential))
470
def get_determine_wants_heads(self, wants, include_tags=False):
472
def determine_wants(refs):
473
potential = set(wants)
475
for k, unpeeled in refs.iteritems():
478
if unpeeled == ZERO_SHA:
480
potential.add(self.source.controldir.get_peeled(k) or unpeeled)
481
return list(potential - self._target_has_shas(potential))
482
return determine_wants
499
484
def _warn_slow(self):
500
485
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
508
493
:param determine_wants: determine_wants callback
509
494
:param mapping: BzrGitMapping to use
510
495
:param limit: Maximum number of commits to import.
511
:return: Tuple with pack hint, last imported revision id and remote
496
:return: Tuple with pack hint, last imported revision id and remote refs
514
498
raise NotImplementedError(self.fetch_objects)
516
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
500
def get_determine_wants_revids(self, revids, include_tags=False):
518
502
for revid in set(revids):
519
503
if self.target.has_revision(revid):
521
505
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
522
506
wants.add(git_sha)
523
return self.get_determine_wants_heads(
524
wants, include_tags=include_tags, tag_selector=tag_selector)
507
return self.get_determine_wants_heads(wants, include_tags=include_tags)
526
509
def fetch(self, revision_id=None, find_ghosts=False,
527
mapping=None, fetch_spec=None, include_tags=False, lossy=False):
510
mapping=None, fetch_spec=None, include_tags=False):
528
511
if mapping is None:
529
512
mapping = self.source.get_mapping()
530
513
if revision_id is not None:
546
529
determine_wants = self.determine_wants_all
548
(pack_hint, _, remote_refs) = self.fetch_objects(
549
determine_wants, mapping, lossy=lossy)
531
(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
550
533
if pack_hint is not None and self.target._format.pack_compresses:
551
534
self.target.pack(hint=pack_hint)
552
result = FetchResult()
553
result.refs = remote_refs
557
538
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
563
544
all_revs = self.target.all_revision_ids()
564
545
parent_map = self.target.get_parent_map(all_revs)
565
546
all_parents = set()
566
for values in viewvalues(parent_map):
567
all_parents.update(values)
547
map(all_parents.update, parent_map.itervalues())
568
548
return set(all_revs) - all_parents
570
550
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
578
558
lambda sha: store[sha].parents)
579
559
wants_recorder = DetermineWantsRecorder(determine_wants)
581
with ui.ui_factory.nested_progress_bar() as pb:
561
pb = ui.ui_factory.nested_progress_bar()
582
563
objects_iter = self.source.fetch_objects(
583
564
wants_recorder, graph_walker, store.get_raw)
584
565
trace.mutter("Importing %d new revisions",
585
566
len(wants_recorder.wants))
586
(pack_hint, last_rev) = import_git_objects(
587
self.target, mapping, objects_iter, store,
588
wants_recorder.wants, pb, limit)
567
(pack_hint, last_rev) = import_git_objects(self.target,
568
mapping, objects_iter, store, wants_recorder.wants, pb,
589
570
return (pack_hint, last_rev, wants_recorder.remote_refs)
592
575
def is_compatible(source, target):
611
594
self._warn_slow()
612
595
remote_refs = self.source.controldir.get_refs_container().as_dict()
613
596
wants = determine_wants(remote_refs)
598
pb = ui.ui_factory.nested_progress_bar()
614
599
target_git_object_retriever = get_object_store(self.target, mapping)
615
with ui.ui_factory.nested_progress_bar() as pb:
616
601
target_git_object_retriever.lock_write()
618
(pack_hint, last_rev) = import_git_objects(
619
self.target, mapping, self.source._git.object_store,
603
(pack_hint, last_rev) = import_git_objects(self.target,
604
mapping, self.source._git.object_store,
620
605
target_git_object_retriever, wants, pb, limit)
621
606
return (pack_hint, last_rev, remote_refs)
623
608
target_git_object_retriever.unlock()
626
613
def is_compatible(source, target):
644
631
raise LossyPushToSameVCS(self.source, self.target)
645
632
old_refs = self.target.controldir.get_refs_container()
648
634
def determine_wants(heads):
649
old_refs = dict([(k, (v, None))
650
for (k, v) in viewitems(heads.as_dict())])
635
old_refs = dict([(k, (v, None)) for (k, v) in heads.as_dict().iteritems()])
651
636
new_refs = update_refs(old_refs)
652
637
ref_changes.update(new_refs)
653
return [sha1 for (sha1, bzr_revid) in viewvalues(new_refs)]
638
return [sha1 for (sha1, bzr_revid) in new_refs.itervalues()]
654
639
self.fetch_objects(determine_wants, lossy=lossy)
655
for k, (git_sha, bzr_revid) in viewitems(ref_changes):
640
for k, (git_sha, bzr_revid) in ref_changes.iteritems():
656
641
self.target._git.refs[k] = git_sha
657
642
new_refs = self.target.controldir.get_refs_container()
658
643
return None, old_refs, new_refs
660
def fetch_objects(self, determine_wants, mapping=None, limit=None,
645
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
662
646
raise NotImplementedError(self.fetch_objects)
664
648
def _target_has_shas(self, shas):
666
[sha for sha in shas if sha in self.target._git.object_store])
649
return set([sha for sha in shas if sha in self.target._git.object_store])
668
651
def fetch(self, revision_id=None, find_ghosts=False,
669
mapping=None, fetch_spec=None, branches=None, limit=None,
670
include_tags=False, lossy=False):
652
mapping=None, fetch_spec=None, branches=None, limit=None, include_tags=False):
671
653
if mapping is None:
672
654
mapping = self.source.get_mapping()
673
655
if revision_id is not None:
681
663
"Unsupported search result type %s" % recipe[0])
683
665
if branches is not None:
684
determine_wants = self.get_determine_wants_branches(
685
branches, include_tags=include_tags)
666
def determine_wants(refs):
668
for name, value in refs.iteritems():
669
if value == ZERO_SHA:
672
if name in branches or (include_tags and is_tag(name)):
686
675
elif fetch_spec is None and revision_id is None:
687
676
determine_wants = self.determine_wants_all
689
determine_wants = self.get_determine_wants_revids(
690
args, include_tags=include_tags)
678
determine_wants = self.get_determine_wants_revids(args, include_tags=include_tags)
691
679
wants_recorder = DetermineWantsRecorder(determine_wants)
692
self.fetch_objects(wants_recorder, mapping, limit=limit, lossy=lossy)
693
result = FetchResult()
694
result.refs = wants_recorder.remote_refs
680
self.fetch_objects(wants_recorder, mapping, limit=limit)
681
return wants_recorder.remote_refs
697
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
683
def get_determine_wants_revids(self, revids, include_tags=False):
699
685
for revid in set(revids):
700
686
if revid == NULL_REVISION:
702
688
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
703
689
wants.add(git_sha)
704
return self.get_determine_wants_heads(wants, include_tags=include_tags, tag_selector=tag_selector)
706
def get_determine_wants_branches(self, branches, include_tags=False):
707
def determine_wants(refs):
709
for name, value in viewitems(refs):
710
if value == ZERO_SHA:
713
if name.endswith(ANNOTATED_TAG_SUFFIX):
716
if name in branches or (include_tags and is_tag(name)):
719
return determine_wants
690
return self.get_determine_wants_heads(wants, include_tags=include_tags)
721
692
def determine_wants_all(self, refs):
723
v for k, v in refs.items()
724
if not v == ZERO_SHA and not k.endswith(ANNOTATED_TAG_SUFFIX)])
693
potential = set([v for v in refs.values() if not v == ZERO_SHA])
725
694
return list(potential - self._target_has_shas(potential))
728
697
class InterLocalGitLocalGitRepository(InterGitGitRepository):
730
def fetch_objects(self, determine_wants, mapping=None, limit=None,
699
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
733
701
raise LossyPushToSameVCS(self.source, self.target)
734
702
if limit is not None:
735
703
raise FetchLimitUnsupported(self)
736
from .remote import DefaultProgressReporter
737
with ui.ui_factory.nested_progress_bar() as pb:
738
progress = DefaultProgressReporter(pb).progress
739
refs = self.source._git.fetch(
740
self.target._git, determine_wants,
704
refs = self.source._git.fetch(self.target._git, determine_wants)
742
705
return (None, None, refs)
751
714
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
753
def fetch_objects(self, determine_wants, mapping=None, limit=None,
716
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
756
718
raise LossyPushToSameVCS(self.source, self.target)
757
719
if limit is not None:
758
720
raise FetchLimitUnsupported(self)
759
721
graphwalker = self.target._git.get_graph_walker()
760
if (CAPABILITY_THIN_PACK in
761
self.source.controldir._client._fetch_capabilities):
722
if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:
762
723
# TODO(jelmer): Avoid reading entire file into memory and
763
724
# only processing it after the whole file has been fetched.