/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/interrepo.py

  • Committer: Jelmer Vernooij
  • Date: 2019-03-04 00:16:27 UTC
  • mfrom: (7293 work)
  • mto: This revision was merged to the branch mainline in revision 7318.
  • Revision ID: jelmer@jelmer.uk-20190304001627-v6u7o6pf97tukhek
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/interrepo.py
79
79
    remote_divergence,
80
80
    )
81
81
from .refs import (
82
 
    get_refs_container,
83
82
    is_tag,
84
83
    )
85
84
from .repository import (
126
125
        raise NotImplementedError(self.fetch_refs)
127
126
 
128
127
    def search_missing_revision_ids(self,
129
 
            find_ghosts=True, revision_ids=None, if_present_ids=None,
130
 
            limit=None):
 
128
                                    find_ghosts=True, revision_ids=None,
 
129
                                    if_present_ids=None, limit=None):
131
130
        if limit is not None:
132
131
            raise FetchLimitUnsupported(self)
133
132
        git_shas = []
145
144
                except KeyError:
146
145
                    raise NoSuchRevision(revid, self.source)
147
146
                git_shas.append(git_sha)
148
 
            walker = Walker(self.source_store,
149
 
                include=git_shas, exclude=[
 
147
            walker = Walker(
 
148
                self.source_store,
 
149
                include=git_shas,
 
150
                exclude=[
150
151
                    sha for sha in self.target.controldir.get_refs_container().as_dict().values()
151
152
                    if sha != ZERO_SHA])
152
153
            missing_revids = set()
153
154
            for entry in walker:
154
 
                for (kind, type_data) in self.source_store.lookup_git_sha(entry.commit.id):
 
155
                for (kind, type_data) in self.source_store.lookup_git_sha(
 
156
                        entry.commit.id):
155
157
                    if kind == "commit":
156
158
                        missing_revids.add(type_data[0])
157
159
            return self.source.revision_ids_to_search_result(missing_revids)
220
222
                new_stop_revids = []
221
223
                for revid in stop_revids:
222
224
                    sha1 = revid_sha_map.get(revid)
223
 
                    if (not revid in missing and
224
 
                        self._revision_needs_fetching(sha1, revid)):
 
225
                    if (revid not in missing and
 
226
                            self._revision_needs_fetching(sha1, revid)):
225
227
                        missing.add(revid)
226
228
                        new_stop_revids.append(revid)
227
229
                stop_revids = set()
240
242
            with Git SHA, Bazaar revid as values.
241
243
        """
242
244
        bzr_refs = {}
243
 
        refs = {}
244
245
        for k in self.target._git.refs.allkeys():
245
246
            try:
246
247
                v = self.target._git.refs.read_ref(k)
250
251
            revid = None
251
252
            if not v.startswith(SYMREF):
252
253
                try:
253
 
                    for (kind, type_data) in self.source_store.lookup_git_sha(v):
254
 
                        if kind == "commit" and self.source.has_revision(type_data[0]):
 
254
                    for (kind, type_data) in self.source_store.lookup_git_sha(
 
255
                            v):
 
256
                        if kind == "commit" and self.source.has_revision(
 
257
                                type_data[0]):
255
258
                            revid = type_data[0]
256
259
                            break
257
260
                except KeyError:
261
264
 
262
265
    def fetch_refs(self, update_refs, lossy, overwrite=False):
263
266
        self._warn_slow()
 
267
        result_refs = {}
264
268
        with self.source_store.lock_read():
265
269
            old_refs = self._get_target_bzr_refs()
266
270
            new_refs = update_refs(old_refs)
267
271
            revidmap = self.fetch_objects(
268
 
                [(git_sha, bzr_revid) for (git_sha, bzr_revid) in new_refs.values() if git_sha is None or not git_sha.startswith(SYMREF)], lossy=lossy)
 
272
                [(git_sha, bzr_revid)
 
273
                 for (git_sha, bzr_revid) in new_refs.values()
 
274
                 if git_sha is None or not git_sha.startswith(SYMREF)],
 
275
                lossy=lossy)
269
276
            for name, (gitid, revid) in viewitems(new_refs):
270
277
                if gitid is None:
271
278
                    try:
273
280
                    except KeyError:
274
281
                        gitid = self.source_store._lookup_revision_sha1(revid)
275
282
                if gitid.startswith(SYMREF):
276
 
                    self.target_refs.set_symbolic_ref(name, gitid[len(SYMREF):])
 
283
                    self.target_refs.set_symbolic_ref(
 
284
                        name, gitid[len(SYMREF):])
277
285
                else:
278
286
                    try:
279
287
                        old_git_id = old_refs[name][0]
281
289
                        self.target_refs.add_if_new(name, gitid)
282
290
                    else:
283
291
                        self.target_refs.set_if_equals(name, old_git_id, gitid)
284
 
        return revidmap, old_refs, new_refs
 
292
                    result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
 
293
        return revidmap, old_refs, result_refs
285
294
 
286
295
    def fetch_objects(self, revs, lossy, limit=None):
287
296
        if not lossy and not self.mapping.roundtripping:
288
297
            for git_sha, bzr_revid in revs:
289
 
                if bzr_revid is not None and needs_roundtripping(self.source, bzr_revid):
 
298
                if (bzr_revid is not None and
 
299
                        needs_roundtripping(self.source, bzr_revid)):
290
300
                    raise NoPushSupport(self.source, self.target, self.mapping,
291
301
                                        bzr_revid)
292
302
        with self.source_store.lock_read():
297
307
                object_generator = MissingObjectsIterator(
298
308
                    self.source_store, self.source, pb)
299
309
                for (old_revid, git_sha) in object_generator.import_revisions(
300
 
                    todo, lossy=lossy):
 
310
                        todo, lossy=lossy):
301
311
                    if lossy:
302
 
                        new_revid = self.mapping.revision_id_foreign_to_bzr(git_sha)
 
312
                        new_revid = self.mapping.revision_id_foreign_to_bzr(
 
313
                            git_sha)
303
314
                    else:
304
315
                        new_revid = old_revid
305
316
                        try:
314
325
                pb.finished()
315
326
 
316
327
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
317
 
            fetch_spec=None, mapped_refs=None):
 
328
              fetch_spec=None, mapped_refs=None):
318
329
        if mapped_refs is not None:
319
330
            stop_revisions = mapped_refs
320
331
        elif revision_id is not None:
324
335
            if recipe[0] in ("search", "proxy-search"):
325
336
                stop_revisions = [(None, revid) for revid in recipe[1]]
326
337
            else:
327
 
                raise AssertionError("Unsupported search result type %s" % recipe[0])
 
338
                raise AssertionError(
 
339
                    "Unsupported search result type %s" % recipe[0])
328
340
        else:
329
 
            stop_revisions = [(None, revid) for revid in self.source.all_revision_ids()]
 
341
            stop_revisions = [(None, revid)
 
342
                              for revid in self.source.all_revision_ids()]
330
343
        self._warn_slow()
331
344
        try:
332
345
            self.fetch_objects(stop_revisions, lossy=False)
348
361
            raise NoPushSupport(self.source, self.target, self.mapping)
349
362
        unpeel_map = UnpeelMap.from_repository(self.source)
350
363
        revidmap = {}
 
364
 
351
365
        def git_update_refs(old_refs):
352
366
            ret = {}
353
 
            self.old_refs = dict([(k, (v, None)) for (k, v) in viewitems(old_refs)])
354
 
            self.new_refs = update_refs(self.old_refs)
355
 
            for name, (gitid, revid) in viewitems(self.new_refs):
 
367
            self.old_refs = {
 
368
                k: (v, None) for (k, v) in viewitems(old_refs)}
 
369
            new_refs = update_refs(self.old_refs)
 
370
            for name, (gitid, revid) in viewitems(new_refs):
356
371
                if gitid is None:
357
372
                    git_sha = self.source_store._lookup_revision_sha1(revid)
358
 
                    gitid = unpeel_map.re_unpeel_tag(git_sha, old_refs.get(name))
 
373
                    gitid = unpeel_map.re_unpeel_tag(
 
374
                        git_sha, old_refs.get(name))
359
375
                if not overwrite:
360
 
                    if remote_divergence(old_refs.get(name), gitid, self.source_store):
 
376
                    if remote_divergence(
 
377
                            old_refs.get(name), gitid, self.source_store):
361
378
                        raise DivergedBranches(self.source, self.target)
362
379
                ret[name] = gitid
363
380
            return ret
364
381
        self._warn_slow()
365
382
        with self.source_store.lock_read():
366
 
            new_refs = self.target.send_pack(git_update_refs,
367
 
                    self.source_store.generate_lossy_pack_data)
 
383
            new_refs = self.target.send_pack(
 
384
                git_update_refs, self.source_store.generate_lossy_pack_data)
368
385
        # FIXME: revidmap?
369
 
        return revidmap, self.old_refs, self.new_refs
 
386
        return revidmap, self.old_refs, new_refs
370
387
 
371
388
    @staticmethod
372
389
    def is_compatible(source, target):
398
415
 
399
416
    def get_determine_wants_heads(self, wants, include_tags=False):
400
417
        wants = set(wants)
 
418
 
401
419
        def determine_wants(refs):
402
 
            potential = set(wants)
 
420
            unpeel_lookup = {}
 
421
            for k, v in viewitems(refs):
 
422
                if k.endswith(ANNOTATED_TAG_SUFFIX):
 
423
                    unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
 
424
            potential = set([unpeel_lookup.get(w, w) for w in wants])
403
425
            if include_tags:
404
 
                for k, unpeeled in viewitems(refs):
 
426
                for k, sha in viewitems(refs):
405
427
                    if k.endswith(ANNOTATED_TAG_SUFFIX):
406
428
                        continue
407
429
                    if not is_tag(k):
408
430
                        continue
409
 
                    if unpeeled == ZERO_SHA:
 
431
                    if sha == ZERO_SHA:
410
432
                        continue
411
 
                    potential.add(unpeeled)
 
433
                    potential.add(sha)
412
434
            return list(potential - self._target_has_shas(potential))
413
435
        return determine_wants
414
436
 
424
446
        self.fetch(revision_id, find_ghosts=False)
425
447
 
426
448
    def search_missing_revision_ids(self,
427
 
            find_ghosts=True, revision_ids=None, if_present_ids=None,
428
 
            limit=None):
 
449
                                    find_ghosts=True, revision_ids=None,
 
450
                                    if_present_ids=None, limit=None):
429
451
        if limit is not None:
430
452
            raise FetchLimitUnsupported(self)
431
453
        if revision_ids is None and if_present_ids is None:
473
495
            potential.add(self.source.controldir.get_peeled(k) or v)
474
496
        return list(potential - self._target_has_shas(potential))
475
497
 
476
 
    def get_determine_wants_heads(self, wants, include_tags=False):
477
 
        wants = set(wants)
478
 
        def determine_wants(refs):
479
 
            potential = set(wants)
480
 
            if include_tags:
481
 
                for k, unpeeled in viewitems(refs):
482
 
                    if not is_tag(k):
483
 
                        continue
484
 
                    if unpeeled == ZERO_SHA:
485
 
                        continue
486
 
                    potential.add(self.source.controldir.get_peeled(k) or unpeeled)
487
 
            return list(potential - self._target_has_shas(potential))
488
 
        return determine_wants
489
 
 
490
498
    def _warn_slow(self):
491
499
        if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
492
500
            trace.warning(
499
507
        :param determine_wants: determine_wants callback
500
508
        :param mapping: BzrGitMapping to use
501
509
        :param limit: Maximum number of commits to import.
502
 
        :return: Tuple with pack hint, last imported revision id and remote refs
 
510
        :return: Tuple with pack hint, last imported revision id and remote
 
511
            refs
503
512
        """
504
513
        raise NotImplementedError(self.fetch_objects)
505
514
 
524
533
                interesting_heads = recipe[1]
525
534
            else:
526
535
                raise AssertionError("Unsupported search result type %s" %
527
 
                        recipe[0])
 
536
                                     recipe[0])
528
537
        else:
529
538
            interesting_heads = None
530
539
 
535
544
            determine_wants = self.determine_wants_all
536
545
 
537
546
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
538
 
            mapping)
 
547
                                                         mapping)
539
548
        if pack_hint is not None and self.target._format.pack_compresses:
540
549
            self.target.pack(hint=pack_hint)
541
550
        return remote_refs
571
580
                    wants_recorder, graph_walker, store.get_raw)
572
581
                trace.mutter("Importing %d new revisions",
573
582
                             len(wants_recorder.wants))
574
 
                (pack_hint, last_rev) = import_git_objects(self.target,
575
 
                    mapping, objects_iter, store, wants_recorder.wants, pb,
576
 
                    limit)
 
583
                (pack_hint, last_rev) = import_git_objects(
 
584
                    self.target, mapping, objects_iter, store,
 
585
                    wants_recorder.wants, pb, limit)
577
586
                return (pack_hint, last_rev, wants_recorder.remote_refs)
578
587
            finally:
579
588
                pb.finished()
601
610
        self._warn_slow()
602
611
        remote_refs = self.source.controldir.get_refs_container().as_dict()
603
612
        wants = determine_wants(remote_refs)
604
 
        create_pb = None
605
613
        pb = ui.ui_factory.nested_progress_bar()
606
614
        target_git_object_retriever = get_object_store(self.target, mapping)
607
615
        try:
608
616
            target_git_object_retriever.lock_write()
609
617
            try:
610
 
                (pack_hint, last_rev) = import_git_objects(self.target,
611
 
                    mapping, self.source._git.object_store,
 
618
                (pack_hint, last_rev) = import_git_objects(
 
619
                    self.target, mapping, self.source._git.object_store,
612
620
                    target_git_object_retriever, wants, pb, limit)
613
621
                return (pack_hint, last_rev, remote_refs)
614
622
            finally:
638
646
            raise LossyPushToSameVCS(self.source, self.target)
639
647
        old_refs = self.target.controldir.get_refs_container()
640
648
        ref_changes = {}
 
649
 
641
650
        def determine_wants(heads):
642
 
            old_refs = dict([(k, (v, None)) for (k, v) in viewitems(heads.as_dict())])
 
651
            old_refs = dict([(k, (v, None))
 
652
                             for (k, v) in viewitems(heads.as_dict())])
643
653
            new_refs = update_refs(old_refs)
644
654
            ref_changes.update(new_refs)
645
655
            return [sha1 for (sha1, bzr_revid) in viewvalues(new_refs)]
649
659
        new_refs = self.target.controldir.get_refs_container()
650
660
        return None, old_refs, new_refs
651
661
 
652
 
    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
 
662
    def fetch_objects(self, determine_wants, mapping=None, limit=None,
 
663
                      lossy=False):
653
664
        raise NotImplementedError(self.fetch_objects)
654
665
 
655
666
    def _target_has_shas(self, shas):
656
 
        return set([sha for sha in shas if sha in self.target._git.object_store])
 
667
        return set(
 
668
            [sha for sha in shas if sha in self.target._git.object_store])
657
669
 
658
670
    def fetch(self, revision_id=None, find_ghosts=False,
659
 
              mapping=None, fetch_spec=None, branches=None, limit=None, include_tags=False):
 
671
              mapping=None, fetch_spec=None, branches=None, limit=None,
 
672
              include_tags=False):
660
673
        if mapping is None:
661
674
            mapping = self.source.get_mapping()
662
675
        if revision_id is not None:
682
695
        elif fetch_spec is None and revision_id is None:
683
696
            determine_wants = self.determine_wants_all
684
697
        else:
685
 
            determine_wants = self.get_determine_wants_revids(args, include_tags=include_tags)
 
698
            determine_wants = self.get_determine_wants_revids(
 
699
                args, include_tags=include_tags)
686
700
        wants_recorder = DetermineWantsRecorder(determine_wants)
687
701
        self.fetch_objects(wants_recorder, mapping, limit=limit)
688
702
        return wants_recorder.remote_refs
705
719
 
706
720
class InterLocalGitLocalGitRepository(InterGitGitRepository):
707
721
 
708
 
    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
 
722
    def fetch_objects(self, determine_wants, mapping=None, limit=None,
 
723
                      lossy=False):
709
724
        if lossy:
710
725
            raise LossyPushToSameVCS(self.source, self.target)
711
726
        if limit is not None:
712
727
            raise FetchLimitUnsupported(self)
713
 
        refs = self.source._git.fetch(self.target._git, determine_wants)
 
728
        from .remote import DefaultProgressReporter
 
729
        pb = ui.ui_factory.nested_progress_bar()
 
730
        progress = DefaultProgressReporter(pb).progress
 
731
        try:
 
732
            refs = self.source._git.fetch(
 
733
                self.target._git, determine_wants,
 
734
                progress=progress)
 
735
        finally:
 
736
            pb.finished()
714
737
        return (None, None, refs)
715
738
 
716
739
    @staticmethod
722
745
 
723
746
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
724
747
 
725
 
    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
 
748
    def fetch_objects(self, determine_wants, mapping=None, limit=None,
 
749
                      lossy=False):
726
750
        if lossy:
727
751
            raise LossyPushToSameVCS(self.source, self.target)
728
752
        if limit is not None:
729
753
            raise FetchLimitUnsupported(self)
730
754
        graphwalker = self.target._git.get_graph_walker()
731
 
        if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:
 
755
        if (CAPABILITY_THIN_PACK in
 
756
                self.source.controldir._client._fetch_capabilities):
732
757
            # TODO(jelmer): Avoid reading entire file into memory and
733
758
            # only processing it after the whole file has been fetched.
734
759
            f = BytesIO()