/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to fetch.py

More work on roundtrip push support.

Show diffs side-by-side

added added

removed removed

Lines of Context:
fetch.py
19
19
    Tag,
20
20
    Tree,
21
21
    S_ISGITLINK,
22
 
    ZERO_SHA,
23
22
    )
24
23
from dulwich.object_store import (
25
24
    tree_lookup_path,
54
53
from bzrlib.revision import (
55
54
    NULL_REVISION,
56
55
    )
57
 
try:
58
 
    from bzrlib.revisiontree import InventoryRevisionTree
59
 
except ImportError: # bzr < 2.4
60
 
    from bzrlib.revisiontree import RevisionTree as InventoryRevisionTree
 
56
from bzrlib.revisiontree import (
 
57
    RevisionTree,
 
58
    )
61
59
from bzrlib.testament import (
62
60
    StrictTestament3,
63
61
    )
79
77
    LRUTreeCache,
80
78
    _tree_to_objects,
81
79
    )
82
 
from bzrlib.plugins.git.refs import extract_tags
83
80
from bzrlib.plugins.git.remote import (
84
81
    RemoteGitRepository,
85
82
    )
154
151
        assert ie.revision is not None
155
152
        if ie.kind == 'symlink':
156
153
            chunks = []
157
 
        else:
 
154
        else: 
158
155
            chunks = blob.chunked
159
156
        texts.insert_record_stream([
160
157
            ChunkedContentFactory((file_id, ie.revision),
263
260
    # Remember for next time
264
261
    existing_children = set()
265
262
    child_modes = {}
266
 
    for name, child_mode, child_hexsha in tree.iteritems():
 
263
    for child_mode, name, child_hexsha in tree.entries():
267
264
        existing_children.add(name)
268
265
        child_path = posixpath.join(path, name)
269
266
        if type(base_tree) is Tree:
278
275
        if stat.S_ISDIR(child_mode):
279
276
            subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
280
277
                child_path, name, (child_base_hexsha, child_hexsha), base_inv,
281
 
                file_id, revision_id, parent_invs, lookup_object,
 
278
                file_id, revision_id, parent_invs, lookup_object, 
282
279
                (child_base_mode, child_mode), store_updater, lookup_file_id,
283
280
                allow_submodules=allow_submodules)
284
281
        elif S_ISGITLINK(child_mode): # submodule
308
305
 
309
306
 
310
307
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
311
 
    o, rev, ret_tree, parent_trees, mapping, unusual_modes, verifiers):
 
308
    o, rev, ret_tree, parent_trees, mapping, unusual_modes):
312
309
    new_unusual_modes = mapping.export_unusual_file_modes(rev)
313
310
    if new_unusual_modes != unusual_modes:
314
311
        raise AssertionError("unusual modes don't match: %r != %r" % (
315
312
            unusual_modes, new_unusual_modes))
316
313
    # Verify that we can reconstruct the commit properly
317
 
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True,
318
 
        verifiers)
 
314
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)
319
315
    if rec_o != o:
320
316
        raise AssertionError("Reconstructed commit differs: %r != %r" % (
321
317
            rec_o, o))
348
344
def import_git_commit(repo, mapping, head, lookup_object,
349
345
                      target_git_object_retriever, trees_cache):
350
346
    o = lookup_object(head)
351
 
    # Note that this uses mapping.revision_id_foreign_to_bzr. If the parents
352
 
    # were bzr roundtripped revisions they would be specified in the
353
 
    # roundtrip data.
354
 
    rev, roundtrip_revid, verifiers = mapping.import_commit(
355
 
        o, mapping.revision_id_foreign_to_bzr)
 
347
    rev, roundtrip_revid, verifiers = mapping.import_commit(o,
 
348
            lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])
356
349
    # We have to do this here, since we have to walk the tree and
357
350
    # we need to make sure to import the blobs / trees with the right
358
351
    # path; this may involve adding them more than once.
384
377
        base_inv = None
385
378
    rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
386
379
              inv_delta, rev.revision_id, rev.parent_ids, base_inv)
387
 
    ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
 
380
    # FIXME: Check verifiers
 
381
    testament = StrictTestament3(rev, inv)
 
382
    calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
388
383
    if roundtrip_revid is not None:
389
384
        original_revid = rev.revision_id
390
385
        rev.revision_id = roundtrip_revid
391
 
        # Check verifiers
392
 
        if getattr(StrictTestament3, "from_revision_tree", None):
393
 
            testament = StrictTestament3(rev, ret_tree)
394
 
        else: # bzr < 2.4
395
 
            testament = StrictTestament3(rev, inv)
396
 
        calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
397
386
        if calculated_verifiers != verifiers:
398
387
            trace.mutter("Testament SHA1 %r for %r did not match %r.",
399
388
                         calculated_verifiers["testament3-sha1"],
400
389
                         rev.revision_id, verifiers["testament3-sha1"])
401
390
            rev.revision_id = original_revid
402
 
    else:
403
 
        calculated_verifiers = {}
404
391
    store_updater.add_object(o, calculated_verifiers, None)
405
392
    store_updater.finish()
 
393
    ret_tree = RevisionTree(repo, inv, rev.revision_id)
406
394
    trees_cache.add(ret_tree)
407
395
    repo.add_revision(rev.revision_id, rev)
408
396
    if "verify" in debug.debug_flags:
409
397
        verify_commit_reconstruction(target_git_object_retriever, 
410
398
            lookup_object, o, rev, ret_tree, parent_trees, mapping,
411
 
            unusual_modes, verifiers)
 
399
            unusual_modes)
412
400
 
413
401
 
414
402
def import_git_objects(repo, mapping, object_iter,
441
429
            continue
442
430
        if isinstance(o, Commit):
443
431
            rev, roundtrip_revid, verifiers = mapping.import_commit(o,
444
 
                mapping.revision_id_foreign_to_bzr)
 
432
                lambda x: None)
445
433
            if (repo.has_revision(rev.revision_id) or
446
434
                (roundtrip_revid and repo.has_revision(roundtrip_revid))):
447
435
                continue
494
482
 
495
483
    _matching_repo_format = GitRepositoryFormat()
496
484
 
497
 
    def _target_has_shas(self, shas):
498
 
        raise NotImplementedError(self._target_has_shas)
499
 
 
500
 
    def get_determine_wants_heads(self, wants, include_tags=False):
501
 
        wants = set(wants)
502
 
        def determine_wants(refs):
503
 
            potential = set(wants)
504
 
            if include_tags:
505
 
                potential.update(
506
 
                    [v[1] or v[0] for v in extract_tags(refs).itervalues()])
507
 
            return list(potential - self._target_has_shas(potential))
508
 
        return determine_wants
509
 
 
510
 
    def determine_wants_all(self, refs):
511
 
        potential = set([sha for (ref, sha) in refs.iteritems() if not
512
 
            ref.endswith("^{}")])
513
 
        return list(potential - self._target_has_shas(potential))
514
 
 
515
485
    @staticmethod
516
486
    def _get_repo_format_to_test():
517
487
        return None
525
495
    """Base InterRepository that copies revisions from a Git into a non-Git
526
496
    repository."""
527
497
 
528
 
    def _target_has_shas(self, shas):
529
 
        revids = [self.source.lookup_foreign_revision_id(sha) for sha in shas]
530
 
        return self.target.has_revisions(revids)
531
 
 
532
 
    def get_determine_wants_revids(self, revids, include_tags=False):
533
 
        wants = set()
534
 
        for revid in set(revids):
535
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
536
 
            wants.add(git_sha)
537
 
        return self.get_determine_wants_heads(wants, include_tags=include_tags)
538
 
 
539
498
    def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
540
499
        """Fetch objects from a remote server.
541
500
 
554
513
        if revision_id is not None:
555
514
            interesting_heads = [revision_id]
556
515
        elif fetch_spec is not None:
557
 
            recipe = fetch_spec.get_recipe()
558
 
            if recipe[0] in ("search", "proxy-search"):
559
 
                interesting_heads = recipe[1]
560
 
            else:
561
 
                raise AssertionError("Unsupported search result type %s" %
562
 
                        recipe[0])
 
516
            interesting_heads = fetch_spec.heads
563
517
        else:
564
518
            interesting_heads = None
565
 
 
566
 
        if interesting_heads is not None:
567
 
            determine_wants = self.get_determine_wants_revids(interesting_heads,
568
 
                include_tags=False)
569
 
        else:
570
 
            determine_wants = self.determine_wants_all
571
 
 
572
 
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
573
 
            mapping, pb)
 
519
        def determine_wants(refs):
 
520
            if interesting_heads is None:
 
521
                ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
 
522
            else:
 
523
                ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
 
524
            return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]
 
525
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)
574
526
        if pack_hint is not None and self.target._format.pack_compresses:
575
527
            self.target.pack(hint=pack_hint)
576
 
        assert isinstance(remote_refs, dict)
577
528
        return remote_refs
578
529
 
579
530
 
618
569
        def progress(text):
619
570
            report_git_progress(pb, text)
620
571
        store = BazaarObjectStore(self.target, mapping)
621
 
        store.lock_write()
 
572
        self.target.lock_write()
622
573
        try:
623
574
            heads = self.get_target_heads()
624
575
            graph_walker = store.get_graph_walker(
632
583
                objects_iter = self.source.fetch_objects(
633
584
                    wants_recorder, graph_walker, store.get_raw,
634
585
                    progress)
635
 
                trace.mutter("Importing %d new revisions",
636
 
                             len(wants_recorder.wants))
637
 
                (pack_hint, last_rev) = import_git_objects(self.target,
638
 
                    mapping, objects_iter, store, wants_recorder.wants, pb,
639
 
                    limit)
 
586
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
 
587
                    objects_iter, store, wants_recorder.wants, pb, limit)
640
588
                return (pack_hint, last_rev, wants_recorder.remote_refs)
641
589
            finally:
642
590
                if create_pb:
643
591
                    create_pb.finished()
644
592
        finally:
645
 
            store.unlock()
 
593
            self.target.unlock()
646
594
 
647
595
    @staticmethod
648
596
    def is_compatible(source, target):
649
597
        """Be compatible with GitRepository."""
650
 
        if not isinstance(source, RemoteGitRepository):
651
 
            return False
652
 
        if not target.supports_rich_root():
653
 
            return False
654
 
        if isinstance(target, GitRepository):
655
 
            return False
656
 
        if not getattr(target._format, "supports_full_versioned_files", True):
657
 
            return False
658
 
        return True
 
598
        return (isinstance(source, RemoteGitRepository) and
 
599
                target.supports_rich_root() and
 
600
                not isinstance(target, GitRepository) and
 
601
                target.texts is not None)
659
602
 
660
603
 
661
604
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
671
614
            create_pb = pb = ui.ui_factory.nested_progress_bar()
672
615
        target_git_object_retriever = BazaarObjectStore(self.target, mapping)
673
616
        try:
674
 
            target_git_object_retriever.lock_write()
 
617
            self.target.lock_write()
675
618
            try:
676
 
                (pack_hint, last_rev) = import_git_objects(self.target,
677
 
                    mapping, self.source._git.object_store,
 
619
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
 
620
                    self.source._git.object_store,
678
621
                    target_git_object_retriever, wants, pb, limit)
679
622
                return (pack_hint, last_rev, remote_refs)
680
623
            finally:
681
 
                target_git_object_retriever.unlock()
 
624
                self.target.unlock()
682
625
        finally:
683
626
            if create_pb:
684
627
                create_pb.finished()
686
629
    @staticmethod
687
630
    def is_compatible(source, target):
688
631
        """Be compatible with GitRepository."""
689
 
        if not isinstance(source, LocalGitRepository):
690
 
            return False
691
 
        if not target.supports_rich_root():
692
 
            return False
693
 
        if isinstance(target, GitRepository):
694
 
            return False
695
 
        if not getattr(target._format, "supports_full_versioned_files", True):
696
 
            return False
697
 
        return True
 
632
        return (isinstance(source, LocalGitRepository) and
 
633
                target.supports_rich_root() and
 
634
                not isinstance(target, GitRepository) and
 
635
                target.texts is not None)
698
636
 
699
637
 
700
638
class InterGitGitRepository(InterGitRepository):
726
664
        else:
727
665
            raise AssertionError
728
666
 
729
 
    def _target_has_shas(self, shas):
730
 
        return set([sha for sha in shas if self.target._git.object_store])
731
 
 
732
667
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
733
668
              mapping=None, fetch_spec=None, branches=None):
734
669
        if mapping is None:
737
672
        if revision_id is not None:
738
673
            args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
739
674
        elif fetch_spec is not None:
740
 
            recipe = fetch_spec.get_recipe()
741
 
            if recipe[0] in ("search", "proxy-search"):
742
 
                heads = recipe[1]
743
 
            else:
744
 
                raise AssertionError("Unsupported search result type %s" % recipe[0])
745
 
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in heads]
 
675
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
746
676
        if branches is not None:
747
 
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store and x[y] != ZERO_SHA]
 
677
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
748
678
        elif fetch_spec is None and revision_id is None:
749
 
            determine_wants = self.determine_wants_all
 
679
            determine_wants = r.object_store.determine_wants_all
750
680
        else:
751
 
            determine_wants = lambda x: [y for y in args if not y in r.object_store and y != ZERO_SHA]
752
 
        wants_recorder = DetermineWantsRecorder(determine_wants)
753
 
        self.fetch_objects(wants_recorder, mapping)
754
 
        return wants_recorder.remote_refs
 
681
            determine_wants = lambda x: [y for y in args if not y in r.object_store]
 
682
        self.fetch_objects(determine_wants, mapping)
755
683
 
756
684
    @staticmethod
757
685
    def is_compatible(source, target):
758
686
        """Be compatible with GitRepository."""
759
687
        return (isinstance(source, GitRepository) and
760
688
                isinstance(target, GitRepository))
761
 
 
762
 
    def get_determine_wants_revids(self, revids, include_tags=False):
763
 
        wants = set()
764
 
        for revid in set(revids):
765
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
766
 
            wants.add(git_sha)
767
 
        return self.get_determine_wants_heads(wants, include_tags=include_tags)