/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to fetch.py

More work on roundtrip push support.

Show diffs side-by-side

added added

removed removed

Lines of Context:
fetch.py
19
19
    Tag,
20
20
    Tree,
21
21
    S_ISGITLINK,
22
 
    ZERO_SHA,
23
22
    )
24
23
from dulwich.object_store import (
25
24
    tree_lookup_path,
54
53
from bzrlib.revision import (
55
54
    NULL_REVISION,
56
55
    )
57
 
try:
58
 
    from bzrlib.revisiontree import InventoryRevisionTree
59
 
except ImportError: # bzr < 2.4
60
 
    from bzrlib.revisiontree import RevisionTree as InventoryRevisionTree
 
56
from bzrlib.revisiontree import (
 
57
    RevisionTree,
 
58
    )
61
59
from bzrlib.testament import (
62
60
    StrictTestament3,
63
61
    )
79
77
    LRUTreeCache,
80
78
    _tree_to_objects,
81
79
    )
82
 
from bzrlib.plugins.git.refs import extract_tags
83
80
from bzrlib.plugins.git.remote import (
84
81
    RemoteGitRepository,
85
82
    )
154
151
        assert ie.revision is not None
155
152
        if ie.kind == 'symlink':
156
153
            chunks = []
157
 
        else:
 
154
        else: 
158
155
            chunks = blob.chunked
159
156
        texts.insert_record_stream([
160
157
            ChunkedContentFactory((file_id, ie.revision),
175
172
 
176
173
 
177
174
class SubmodulesRequireSubtrees(BzrError):
178
 
    _fmt = ("The repository you are fetching from contains submodules. "
179
 
            "To continue, upgrade your Bazaar repository to a format that "
180
 
            "supports nested trees, such as 'development-subtree'.")
 
175
    _fmt = """The repository you are fetching from contains submodules. To continue, upgrade your Bazaar repository to a format that supports nested trees, such as 'development-subtree'."""
181
176
    internal = False
182
177
 
183
178
 
184
179
def import_git_submodule(texts, mapping, path, name, (base_hexsha, hexsha),
185
180
    base_inv, parent_id, revision_id, parent_invs, lookup_object,
186
181
    (base_mode, mode), store_updater, lookup_file_id):
187
 
    """Import a git submodule."""
188
182
    if base_hexsha == hexsha and base_mode == mode:
189
183
        return [], {}
190
184
    file_id = lookup_file_id(path)
266
260
    # Remember for next time
267
261
    existing_children = set()
268
262
    child_modes = {}
269
 
    for name, child_mode, child_hexsha in tree.iteritems():
 
263
    for child_mode, name, child_hexsha in tree.entries():
270
264
        existing_children.add(name)
271
265
        child_path = posixpath.join(path, name)
272
266
        if type(base_tree) is Tree:
281
275
        if stat.S_ISDIR(child_mode):
282
276
            subinvdelta, grandchildmodes = import_git_tree(texts, mapping,
283
277
                child_path, name, (child_base_hexsha, child_hexsha), base_inv,
284
 
                file_id, revision_id, parent_invs, lookup_object,
 
278
                file_id, revision_id, parent_invs, lookup_object, 
285
279
                (child_base_mode, child_mode), store_updater, lookup_file_id,
286
280
                allow_submodules=allow_submodules)
287
281
        elif S_ISGITLINK(child_mode): # submodule
292
286
                file_id, revision_id, parent_invs, lookup_object,
293
287
                (child_base_mode, child_mode), store_updater, lookup_file_id)
294
288
        else:
295
 
            if not mapping.is_special_file(name):
296
 
                subinvdelta = import_git_blob(texts, mapping, child_path, name,
297
 
                    (child_base_hexsha, child_hexsha), base_inv, file_id,
298
 
                    revision_id, parent_invs, lookup_object,
299
 
                    (child_base_mode, child_mode), store_updater, lookup_file_id)
300
 
            else:
301
 
                subinvdelta = []
 
289
            subinvdelta = import_git_blob(texts, mapping, child_path, name,
 
290
                (child_base_hexsha, child_hexsha), base_inv, file_id,
 
291
                revision_id, parent_invs, lookup_object,
 
292
                (child_base_mode, child_mode), store_updater, lookup_file_id)
302
293
            grandchildmodes = {}
303
294
        child_modes.update(grandchildmodes)
304
295
        invdelta.extend(subinvdelta)
314
305
 
315
306
 
316
307
def verify_commit_reconstruction(target_git_object_retriever, lookup_object,
317
 
    o, rev, ret_tree, parent_trees, mapping, unusual_modes, verifiers):
 
308
    o, rev, ret_tree, parent_trees, mapping, unusual_modes):
318
309
    new_unusual_modes = mapping.export_unusual_file_modes(rev)
319
310
    if new_unusual_modes != unusual_modes:
320
311
        raise AssertionError("unusual modes don't match: %r != %r" % (
321
312
            unusual_modes, new_unusual_modes))
322
313
    # Verify that we can reconstruct the commit properly
323
 
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True,
324
 
        verifiers)
 
314
    rec_o = target_git_object_retriever._reconstruct_commit(rev, o.tree, True)
325
315
    if rec_o != o:
326
316
        raise AssertionError("Reconstructed commit differs: %r != %r" % (
327
317
            rec_o, o))
328
318
    diff = []
329
319
    new_objs = {}
330
320
    for path, obj, ie in _tree_to_objects(ret_tree, parent_trees,
331
 
        target_git_object_retriever._cache.idmap, unusual_modes,
332
 
        mapping.BZR_DUMMY_FILE):
 
321
        target_git_object_retriever._cache.idmap, unusual_modes, mapping.BZR_DUMMY_FILE):
333
322
        old_obj_id = tree_lookup_path(lookup_object, o.tree, path)[1]
334
323
        new_objs[path] = obj
335
324
        if obj.id != old_obj_id:
355
344
def import_git_commit(repo, mapping, head, lookup_object,
356
345
                      target_git_object_retriever, trees_cache):
357
346
    o = lookup_object(head)
358
 
    # Note that this uses mapping.revision_id_foreign_to_bzr. If the parents
359
 
    # were bzr roundtripped revisions they would be specified in the
360
 
    # roundtrip data.
361
 
    rev, roundtrip_revid, verifiers = mapping.import_commit(
362
 
        o, mapping.revision_id_foreign_to_bzr)
363
 
    if roundtrip_revid is not None:
364
 
        original_revid = rev.revision_id
365
 
        rev.revision_id = roundtrip_revid
 
347
    rev, roundtrip_revid, verifiers = mapping.import_commit(o,
 
348
            lambda x: target_git_object_retriever.lookup_git_sha(x)[1][0])
366
349
    # We have to do this here, since we have to walk the tree and
367
350
    # we need to make sure to import the blobs / trees with the right
368
351
    # path; this may involve adding them more than once.
376
359
        base_tree = lookup_object(o.parents[0]).tree
377
360
        base_mode = stat.S_IFDIR
378
361
    store_updater = target_git_object_retriever._get_updater(rev)
379
 
    tree_supplement = mapping.get_fileid_map(lookup_object, o.tree)
 
362
    fileid_map = mapping.get_fileid_map(lookup_object, o.tree)
380
363
    inv_delta, unusual_modes = import_git_tree(repo.texts,
381
364
            mapping, "", "", (base_tree, o.tree), base_inv,
382
365
            None, rev.revision_id, [p.inventory for p in parent_trees],
383
366
            lookup_object, (base_mode, stat.S_IFDIR), store_updater,
384
 
            tree_supplement.lookup_file_id,
385
 
            allow_submodules=getattr(repo._format, "supports_tree_reference",
386
 
                False))
 
367
            fileid_map.lookup_file_id,
 
368
            allow_submodules=getattr(repo._format, "supports_tree_reference", False))
387
369
    if unusual_modes != {}:
388
370
        for path, mode in unusual_modes.iteritems():
389
371
            warn_unusual_mode(rev.foreign_revid, path, mode)
395
377
        base_inv = None
396
378
    rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
397
379
              inv_delta, rev.revision_id, rev.parent_ids, base_inv)
398
 
    ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
399
 
    # Check verifiers
400
 
    if verifiers and roundtrip_revid is not None:
401
 
        if getattr(StrictTestament3, "from_revision_tree", None):
402
 
            testament = StrictTestament3(rev, ret_tree)
403
 
        else: # bzr < 2.4
404
 
            testament = StrictTestament3(rev, inv)
405
 
        calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
 
380
    # FIXME: Check verifiers
 
381
    testament = StrictTestament3(rev, inv)
 
382
    calculated_verifiers = { "testament3-sha1": testament.as_sha1() }
 
383
    if roundtrip_revid is not None:
 
384
        original_revid = rev.revision_id
 
385
        rev.revision_id = roundtrip_revid
406
386
        if calculated_verifiers != verifiers:
407
387
            trace.mutter("Testament SHA1 %r for %r did not match %r.",
408
388
                         calculated_verifiers["testament3-sha1"],
409
389
                         rev.revision_id, verifiers["testament3-sha1"])
410
390
            rev.revision_id = original_revid
411
 
            rev.inventory_sha1, inv = repo.add_inventory_by_delta(basis_id,
412
 
              inv_delta, rev.revision_id, rev.parent_ids, base_inv)
413
 
            ret_tree = InventoryRevisionTree(repo, inv, rev.revision_id)
414
 
    else:
415
 
        calculated_verifiers = {}
416
391
    store_updater.add_object(o, calculated_verifiers, None)
417
392
    store_updater.finish()
 
393
    ret_tree = RevisionTree(repo, inv, rev.revision_id)
418
394
    trees_cache.add(ret_tree)
419
395
    repo.add_revision(rev.revision_id, rev)
420
396
    if "verify" in debug.debug_flags:
421
397
        verify_commit_reconstruction(target_git_object_retriever, 
422
398
            lookup_object, o, rev, ret_tree, parent_trees, mapping,
423
 
            unusual_modes, verifiers)
 
399
            unusual_modes)
424
400
 
425
401
 
426
402
def import_git_objects(repo, mapping, object_iter,
453
429
            continue
454
430
        if isinstance(o, Commit):
455
431
            rev, roundtrip_revid, verifiers = mapping.import_commit(o,
456
 
                mapping.revision_id_foreign_to_bzr)
 
432
                lambda x: None)
457
433
            if (repo.has_revision(rev.revision_id) or
458
434
                (roundtrip_revid and repo.has_revision(roundtrip_revid))):
459
435
                continue
506
482
 
507
483
    _matching_repo_format = GitRepositoryFormat()
508
484
 
509
 
    def _target_has_shas(self, shas):
510
 
        raise NotImplementedError(self._target_has_shas)
511
 
 
512
 
    def get_determine_wants_heads(self, wants, include_tags=False):
513
 
        wants = set(wants)
514
 
        def determine_wants(refs):
515
 
            potential = set(wants)
516
 
            if include_tags:
517
 
                potential.update(
518
 
                    [v[1] or v[0] for v in extract_tags(refs).itervalues()])
519
 
            return list(potential - self._target_has_shas(potential))
520
 
        return determine_wants
521
 
 
522
 
    def determine_wants_all(self, refs):
523
 
        potential = set([sha for (ref, sha) in refs.iteritems() if not
524
 
            ref.endswith("^{}")])
525
 
        return list(potential - self._target_has_shas(potential))
526
 
 
527
485
    @staticmethod
528
486
    def _get_repo_format_to_test():
529
487
        return None
537
495
    """Base InterRepository that copies revisions from a Git into a non-Git
538
496
    repository."""
539
497
 
540
 
    def _target_has_shas(self, shas):
541
 
        revids = [self.source.lookup_foreign_revision_id(sha) for sha in shas]
542
 
        return self.target.has_revisions(revids)
543
 
 
544
 
    def get_determine_wants_revids(self, revids, include_tags=False):
545
 
        wants = set()
546
 
        for revid in set(revids):
547
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
548
 
            wants.add(git_sha)
549
 
        return self.get_determine_wants_heads(wants,
550
 
            include_tags=include_tags)
551
 
 
552
498
    def fetch_objects(self, determine_wants, mapping, pb=None, limit=None):
553
499
        """Fetch objects from a remote server.
554
500
 
567
513
        if revision_id is not None:
568
514
            interesting_heads = [revision_id]
569
515
        elif fetch_spec is not None:
570
 
            recipe = fetch_spec.get_recipe()
571
 
            if recipe[0] in ("search", "proxy-search"):
572
 
                interesting_heads = recipe[1]
573
 
            else:
574
 
                raise AssertionError("Unsupported search result type %s" %
575
 
                        recipe[0])
 
516
            interesting_heads = fetch_spec.heads
576
517
        else:
577
518
            interesting_heads = None
578
 
 
579
 
        if interesting_heads is not None:
580
 
            determine_wants = self.get_determine_wants_revids(
581
 
                interesting_heads, include_tags=False)
582
 
        else:
583
 
            determine_wants = self.determine_wants_all
584
 
 
585
 
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
586
 
            mapping, pb)
 
519
        def determine_wants(refs):
 
520
            if interesting_heads is None:
 
521
                ret = [sha for (ref, sha) in refs.iteritems() if not ref.endswith("^{}")]
 
522
            else:
 
523
                ret = [self.source.lookup_bzr_revision_id(revid)[0] for revid in interesting_heads if revid not in (None, NULL_REVISION)]
 
524
            return [rev for rev in ret if not self.target.has_revision(self.source.lookup_foreign_revision_id(rev))]
 
525
        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants, mapping, pb)
587
526
        if pack_hint is not None and self.target._format.pack_compresses:
588
527
            self.target.pack(hint=pack_hint)
589
 
        assert isinstance(remote_refs, dict)
590
528
        return remote_refs
591
529
 
592
530
 
631
569
        def progress(text):
632
570
            report_git_progress(pb, text)
633
571
        store = BazaarObjectStore(self.target, mapping)
634
 
        store.lock_write()
 
572
        self.target.lock_write()
635
573
        try:
636
574
            heads = self.get_target_heads()
637
575
            graph_walker = store.get_graph_walker(
645
583
                objects_iter = self.source.fetch_objects(
646
584
                    wants_recorder, graph_walker, store.get_raw,
647
585
                    progress)
648
 
                trace.mutter("Importing %d new revisions",
649
 
                             len(wants_recorder.wants))
650
 
                (pack_hint, last_rev) = import_git_objects(self.target,
651
 
                    mapping, objects_iter, store, wants_recorder.wants, pb,
652
 
                    limit)
 
586
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
 
587
                    objects_iter, store, wants_recorder.wants, pb, limit)
653
588
                return (pack_hint, last_rev, wants_recorder.remote_refs)
654
589
            finally:
655
590
                if create_pb:
656
591
                    create_pb.finished()
657
592
        finally:
658
 
            store.unlock()
 
593
            self.target.unlock()
659
594
 
660
595
    @staticmethod
661
596
    def is_compatible(source, target):
662
597
        """Be compatible with GitRepository."""
663
 
        if not isinstance(source, RemoteGitRepository):
664
 
            return False
665
 
        if not target.supports_rich_root():
666
 
            return False
667
 
        if isinstance(target, GitRepository):
668
 
            return False
669
 
        if not getattr(target._format, "supports_full_versioned_files", True):
670
 
            return False
671
 
        return True
 
598
        return (isinstance(source, RemoteGitRepository) and
 
599
                target.supports_rich_root() and
 
600
                not isinstance(target, GitRepository) and
 
601
                target.texts is not None)
672
602
 
673
603
 
674
604
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
684
614
            create_pb = pb = ui.ui_factory.nested_progress_bar()
685
615
        target_git_object_retriever = BazaarObjectStore(self.target, mapping)
686
616
        try:
687
 
            target_git_object_retriever.lock_write()
 
617
            self.target.lock_write()
688
618
            try:
689
 
                (pack_hint, last_rev) = import_git_objects(self.target,
690
 
                    mapping, self.source._git.object_store,
 
619
                (pack_hint, last_rev) = import_git_objects(self.target, mapping,
 
620
                    self.source._git.object_store,
691
621
                    target_git_object_retriever, wants, pb, limit)
692
622
                return (pack_hint, last_rev, remote_refs)
693
623
            finally:
694
 
                target_git_object_retriever.unlock()
 
624
                self.target.unlock()
695
625
        finally:
696
626
            if create_pb:
697
627
                create_pb.finished()
699
629
    @staticmethod
700
630
    def is_compatible(source, target):
701
631
        """Be compatible with GitRepository."""
702
 
        if not isinstance(source, LocalGitRepository):
703
 
            return False
704
 
        if not target.supports_rich_root():
705
 
            return False
706
 
        if isinstance(target, GitRepository):
707
 
            return False
708
 
        if not getattr(target._format, "supports_full_versioned_files", True):
709
 
            return False
710
 
        return True
 
632
        return (isinstance(source, LocalGitRepository) and
 
633
                target.supports_rich_root() and
 
634
                not isinstance(target, GitRepository) and
 
635
                target.texts is not None)
711
636
 
712
637
 
713
638
class InterGitGitRepository(InterGitRepository):
739
664
        else:
740
665
            raise AssertionError
741
666
 
742
 
    def _target_has_shas(self, shas):
743
 
        return set([sha for sha in shas if self.target._git.object_store])
744
 
 
745
667
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
746
668
              mapping=None, fetch_spec=None, branches=None):
747
669
        if mapping is None:
748
670
            mapping = self.source.get_mapping()
749
671
        r = self.target._git
750
672
        if revision_id is not None:
751
 
            args = [self.source.lookup_bzr_revision_id(revision_id)[0]]
 
673
            args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
752
674
        elif fetch_spec is not None:
753
 
            recipe = fetch_spec.get_recipe()
754
 
            if recipe[0] in ("search", "proxy-search"):
755
 
                heads = recipe[1]
756
 
            else:
757
 
                raise AssertionError(
758
 
                    "Unsupported search result type %s" % recipe[0])
759
 
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in
760
 
                    heads]
 
675
            args = [mapping.revision_id_bzr_to_foreign(revid)[0] for revid in fetch_spec.heads]
761
676
        if branches is not None:
762
 
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store and x[y] != ZERO_SHA]
 
677
            determine_wants = lambda x: [x[y] for y in branches if not x[y] in r.object_store]
763
678
        elif fetch_spec is None and revision_id is None:
764
 
            determine_wants = self.determine_wants_all
 
679
            determine_wants = r.object_store.determine_wants_all
765
680
        else:
766
 
            determine_wants = lambda x: [y for y in args if not y in r.object_store and y != ZERO_SHA]
767
 
        wants_recorder = DetermineWantsRecorder(determine_wants)
768
 
        self.fetch_objects(wants_recorder, mapping)
769
 
        return wants_recorder.remote_refs
 
681
            determine_wants = lambda x: [y for y in args if not y in r.object_store]
 
682
        self.fetch_objects(determine_wants, mapping)
770
683
 
771
684
    @staticmethod
772
685
    def is_compatible(source, target):
773
686
        """Be compatible with GitRepository."""
774
687
        return (isinstance(source, GitRepository) and
775
688
                isinstance(target, GitRepository))
776
 
 
777
 
    def get_determine_wants_revids(self, revids, include_tags=False):
778
 
        wants = set()
779
 
        for revid in set(revids):
780
 
            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
781
 
            wants.add(git_sha)
782
 
        return self.get_determine_wants_heads(wants,
783
 
            include_tags=include_tags)