/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to fetch.py

More work on roundtrip push support.

Show diffs side-by-side

added added

removed removed

Lines of Context:
fetch.py
19
19
    Tag,
20
20
    Tree,
21
21
    S_ISGITLINK,
22
 
    ZERO_SHA,
23
22
    )
24
23
from dulwich.object_store import (
25
24
    tree_lookup_path,
54
53
from bzrlib.revision import (
55
54
    NULL_REVISION,
56
55
    )
57
 
try:
58
 
    from bzrlib.revisiontree import InventoryRevisionTree
59
 
except ImportError: # bzr < 2.4
60
 
    from bzrlib.revisiontree import RevisionTree as InventoryRevisionTree
 
56
from bzrlib.revisiontree import (
 
57
    RevisionTree,
 
58
    )
61
59
from bzrlib.testament import (
62
60
    StrictTestament3,
63
61
    )
79
77
    LRUTreeCache,
80
78
    _tree_to_objects,
81
79
    )
82
 
from bzrlib.plugins.git.refs import extract_tags
83
80
from bzrlib.plugins.git.remote import (
84
81
    RemoteGitRepository,
85
82
    )
263
260
    # Remember for next time
264
261
    existing_children = set()
265
262
    child_modes = {}
266
 
    for name, child_mode, child_hexsha in tree.iteritems():
 
263
    for child_mode, name, child_hexsha in tree.entries():
267
264
        existing_children.add(name)
268
265
        child_path = posixpath.join(path, name)
269
266
        if type(base_tree) is Tree:
278
275
        if stat.S_ISDIR(child_mode):
279
276
            subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
280
277
                child_path, name, (child_base_hexsha, child_hexsha), base_inv,
281
 
                file_id, revision_id, parent_invs, lookup_object,
 
278
                file_id, revision_id, parent_invs, lookup_object, 
282
279
                (child_base_mode, child_mode), store_updater, lookup_file_id,
283
280
                allow_submodules=allow_submodules)
284
281
        elif S_ISGITLINK(child_mode): # submodule
308
305
 
309
306
 
310
307
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
311
 
    o, rev, ret_tree, parent_trees, mapping, unusual_modes, verifiers):
 
308
    o, rev, ret_tree, parent_trees, mapping, unusual_modes):
312
309
    new_unusual_modes = mapping.export_unusual_file_modes(rev)
313
310
    if new_unusual_modes != unusual_modes:
314
311
        raise AssertionError("unusual modes don't match: %r != %r" % (
315
312
            unusual_modes, new_unusual_modes))
316
313
    # Verify that we can reconstruct the commit properly
317
 
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True,
318
 
        verifiers)
 
314
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)
319
315
    if rec_o != o:
320
316
        raise AssertionError("Reconstructed commit differs: %r != %r" % (
321
317
            rec_o, o))
348
344
def import_git_commit(repo, mapping, head, lookup_object,
349
345
                      target_git_object_retriever, trees_cache):
350
346
    o = lookup_object(head)
351
 
    # Note that this uses mapping.revision_id_foreign_to_bzr. If the parents
352
 
    # were bzr roundtripped revisions they would be specified in the
353
 
    # roundtrip data.
354
 
    rev, roundtrip_revid, verifiers = mapping.import_commit(
355
 
        o, mapping.revision_id_foreign_to_bzr)
 
347
    rev, roundtrip_revid, verifiers = mapping.import_commit(o,
 
348
            lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])
356
349
    # We have to do this here, since we have to walk the tree and
357
350
    # we need to make sure to import the blobs / trees with the right
358
351
    # path; this may involve adding them more than once.
384
377
        base_inv = None
385
378
    rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
386
379
              inv_delta, rev.revision_id, rev.parent_ids, base_inv)
387
 
    ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
 
380
    # FIXME: Check verifiers
 
381
    testament = StrictTestament3(rev, inv)
 
382
    calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
388
383
    if roundtrip_revid is not None:
389
384
        original_revid = rev.revision_id
390
385
        rev.revision_id = roundtrip_revid
391
 
        # Check verifiers
392
 
        if getattr(StrictTestament3, "from_revision_tree", None):
393
 
            testament = StrictTestament3(rev, ret_tree)
394
 
        else: # bzr < 2.4
395
 
            testament = StrictTestament3(rev, inv)
396
 
        calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
397
386
        if calculated_verifiers != verifiers:
398
387
            trace.mutter("Testament SHA1 %r for %r did not match %r.",
399
388
                         calculated_verifiers["testament3-sha1"],
400
389
                         rev.revision_id, verifiers["testament3-sha1"])
401
390
            rev.revision_id = original_revid
402
 
    else:
403
 
        calculated_verifiers = {}
404
391
    store_updater.add_object(o, calculated_verifiers, None)
405
392
    store_updater.finish()
 
393
    ret_tree = RevisionTree(repo, inv, rev.revision_id)
406
394
    trees_cache.add(ret_tree)
407
395
    repo.add_revision(rev.revision_id, rev)
408
396
    if "verify" in debug.debug_flags:
409
397
        verify_commit_reconstruction(target_git_object_retriever, 
410
398
            lookup_object, o, rev, ret_tree, parent_trees, mapping,
411
 
            unusual_modes, verifiers)
 
399
            unusual_modes)
412
400
 
413
401
 
414
402
def import_git_objects(repo, mapping, object_iter,
441
429
            continue
442
430
        if isinstance(o, Commit):
443
431
            rev, roundtrip_revid, verifiers = mapping.import_commit(o,
444
 
                mapping.revision_id_foreign_to_bzr)
 
432
                lambda x: None)
445
433
            if (repo.has_revision(rev.revision_id) or
446
434
                (roundtrip_revid and repo.has_revision(roundtrip_revid))):
447
435
                continue
494
482
 
495
483
    _matching_repo_format = GitRepositoryFormat()
496
484
 
497
 
    def _target_has_shas(self, shas):
498
 
        raise NotImplementedError(self._target_has_shas)
499
 
 
500
 
    def get_determine_wants_heads(self, wants, include_tags=False):
501
 
        wants = set(wants)
502
 
        def determine_wants(refs):
503
 
            potential = set(wants)
504
 
            if include_tags:
505
 
                potential.update([v[1] or v[0] for v in extract_tags(refs).itervalues()])
506
 
            return list(potential - self._target_has_shas(potential))
507
 
        return determine_wants
508
 
 
509
 
    def determine_wants_all(self, refs):
510
 
        potential = set([sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")])
511
 
        return list(potential - self._target_has_shas(potential))
512
 
 
513
485
    @staticmethod
514
486
    def _get_repo_format_to_test():
515
487
        return None
523
495
    """Base InterRepository that copies revisions from a Git into a non-Git
524
496
    repository."""
525
497
 
526
 
    def _target_has_shas(self, shas):
527
 
        revids = [self.source.lookup_foreign_revision_id(sha) for sha in shas]
528
 
        return self.target.has_revisions(revids)
529
 
 
530
 
    def get_determine_wants_revids(self, revids, include_tags=False):
531
 
        wants = set()
532
 
        for revid in set(revids):
533
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
534
 
            wants.add(git_sha)
535
 
        return self.get_determine_wants_heads(wants, include_tags=include_tags)
536
 
 
537
498
    def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
538
499
        """Fetch objects from a remote server.
539
500
 
552
513
        if revision_id is not None:
553
514
            interesting_heads = [revision_id]
554
515
        elif fetch_spec is not None:
555
 
            recipe = fetch_spec.get_recipe()
556
 
            if recipe[0] in ("search", "proxy-search"):
557
 
                interesting_heads = recipe[1]
558
 
            else:
559
 
                raise AssertionError("Unsupported search result type %s" % recipe[0])
 
516
            interesting_heads = fetch_spec.heads
560
517
        else:
561
518
            interesting_heads = None
562
 
 
563
 
        if interesting_heads is not None:
564
 
            determine_wants = self.get_determine_wants_revids(interesting_heads,
565
 
                include_tags=False)
566
 
        else:
567
 
            determine_wants = self.determine_wants_all
568
 
 
569
 
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
570
 
            mapping, pb)
 
519
        def determine_wants(refs):
 
520
            if interesting_heads is None:
 
521
                ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
 
522
            else:
 
523
                ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
 
524
            return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]
 
525
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)
571
526
        if pack_hint is not None and self.target._format.pack_compresses:
572
527
            self.target.pack(hint=pack_hint)
573
 
        assert isinstance(remote_refs, dict)
574
528
        return remote_refs
575
529
 
576
530
 
615
569
        def progress(text):
616
570
            report_git_progress(pb, text)
617
571
        store = BazaarObjectStore(self.target, mapping)
618
 
        store.lock_write()
 
572
        self.target.lock_write()
619
573
        try:
620
574
            heads = self.get_target_heads()
621
575
            graph_walker = store.get_graph_walker(
629
583
                objects_iter = self.source.fetch_objects(
630
584
                    wants_recorder, graph_walker, store.get_raw,
631
585
                    progress)
632
 
                trace.mutter("Importing %d new revisions", len(wants_recorder.wants))
633
586
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
634
587
                    objects_iter, store, wants_recorder.wants, pb, limit)
635
588
                return (pack_hint, last_rev, wants_recorder.remote_refs)
637
590
                if create_pb:
638
591
                    create_pb.finished()
639
592
        finally:
640
 
            store.unlock()
 
593
            self.target.unlock()
641
594
 
642
595
    @staticmethod
643
596
    def is_compatible(source, target):
644
597
        """Be compatible with GitRepository."""
645
 
        if not isinstance(source, RemoteGitRepository):
646
 
            return False
647
 
        if not target.supports_rich_root():
648
 
            return False
649
 
        if isinstance(target, GitRepository):
650
 
            return False
651
 
        if not target._format.supports_full_versioned_files:
652
 
            return False
653
 
        return True
 
598
        return (isinstance(source, RemoteGitRepository) and
 
599
                target.supports_rich_root() and
 
600
                not isinstance(target, GitRepository) and
 
601
                target.texts is not None)
654
602
 
655
603
 
656
604
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
666
614
            create_pb = pb = ui.ui_factory.nested_progress_bar()
667
615
        target_git_object_retriever = BazaarObjectStore(self.target, mapping)
668
616
        try:
669
 
            target_git_object_retriever.lock_write()
 
617
            self.target.lock_write()
670
618
            try:
671
619
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
672
620
                    self.source._git.object_store,
673
621
                    target_git_object_retriever, wants, pb, limit)
674
622
                return (pack_hint, last_rev, remote_refs)
675
623
            finally:
676
 
                target_git_object_retriever.unlock()
 
624
                self.target.unlock()
677
625
        finally:
678
626
            if create_pb:
679
627
                create_pb.finished()
681
629
    @staticmethod
682
630
    def is_compatible(source, target):
683
631
        """Be compatible with GitRepository."""
684
 
        if not isinstance(source, LocalGitRepository):
685
 
            return False
686
 
        if not target.supports_rich_root():
687
 
            return False
688
 
        if isinstance(target, GitRepository):
689
 
            return False
690
 
        if not getattr(target._format, "supports_full_versioned_files", True):
691
 
            return False
692
 
        return True
 
632
        return (isinstance(source, LocalGitRepository) and
 
633
                target.supports_rich_root() and
 
634
                not isinstance(target, GitRepository) and
 
635
                target.texts is not None)
693
636
 
694
637
 
695
638
class InterGitGitRepository(InterGitRepository):
721
664
        else:
722
665
            raise AssertionError
723
666
 
724
 
    def _target_has_shas(self, shas):
725
 
        return set([sha for sha in shas if self.target._git.object_store])
726
 
 
727
667
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
728
668
              mapping=None, fetch_spec=None, branches=None):
729
669
        if mapping is None:
732
672
        if revision_id is not None:
733
673
            args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
734
674
        elif fetch_spec is not None:
735
 
            recipe = fetch_spec.get_recipe()
736
 
            if recipe[0] in ("search", "proxy-search"):
737
 
                heads = recipe[1]
738
 
            else:
739
 
                raise AssertionError("Unsupported search result type %s" % recipe[0])
740
 
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in heads]
 
675
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
741
676
        if branches is not None:
742
 
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store and x[y] != ZERO_SHA]
 
677
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
743
678
        elif fetch_spec is None and revision_id is None:
744
 
            determine_wants = self.determine_wants_all
 
679
            determine_wants = r.object_store.determine_wants_all
745
680
        else:
746
 
            determine_wants = lambda x: [y for y in args if not y in r.object_store and y != ZERO_SHA]
747
 
        wants_recorder = DetermineWantsRecorder(determine_wants)
748
 
        self.fetch_objects(wants_recorder, mapping)
749
 
        return wants_recorder.remote_refs
 
681
            determine_wants = lambda x: [y for y in args if not y in r.object_store]
 
682
        self.fetch_objects(determine_wants, mapping)
750
683
 
751
684
    @staticmethod
752
685
    def is_compatible(source, target):
753
686
        """Be compatible with GitRepository."""
754
687
        return (isinstance(source, GitRepository) and
755
688
                isinstance(target, GitRepository))
756
 
 
757
 
    def get_determine_wants_revids(self, revids, include_tags=False):
758
 
        wants = set()
759
 
        for revid in set(revids):
760
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
761
 
            wants.add(git_sha)
762
 
        return self.get_determine_wants_heads(wants, include_tags=include_tags)