/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to object_store.py

More work on roundtrip push support.

Show diffs side-by-side

added added

removed removed

Lines of Context:
object_store.py
21
21
    Commit,
22
22
    Tree,
23
23
    sha_to_hex,
24
 
    ZERO_SHA,
25
24
    )
26
25
from dulwich.object_store import (
27
26
    BaseObjectStore,
34
33
    ui,
35
34
    urlutils,
36
35
    )
37
 
from bzrlib.lock import LogicalLockResult
38
36
from bzrlib.revision import (
39
37
    NULL_REVISION,
40
38
    )
60
58
def get_object_store(repo, mapping=None):
61
59
    git = getattr(repo, "_git", None)
62
60
    if git is not None:
63
 
        git.object_store.unlock = lambda x: None
64
 
        git.object_store.lock_read = LogicalLockResult(lambda: None)
65
 
        git.object_store.lock_write = LogicalLockResult(lambda: None)
66
61
        return git.object_store
67
62
    return BazaarObjectStore(repo, mapping)
68
63
 
99
94
                todo.append(revid)
100
95
            else:
101
96
                assert tree.get_revision_id() == revid
 
97
                assert tree.inventory.revision_id == revid
102
98
                trees[revid] = tree
103
99
        for tree in self.repository.revision_trees(todo):
104
100
            trees[tree.get_revision_id()] = tree
112
108
        self._cache.add(tree.get_revision_id(), tree)
113
109
 
114
110
 
115
 
def _find_missing_bzr_revids(graph, want, have):
 
111
def _find_missing_bzr_revids(get_parent_map, want, have):
116
112
    """Find the revisions that have to be pushed.
117
113
 
118
114
    :param get_parent_map: Function that returns the parents for a sequence
121
117
    :param have: Revisions the target already has
122
118
    :return: Set of revisions to fetch
123
119
    """
 
120
    pending = want - have
 
121
    processed = set()
124
122
    todo = set()
125
 
    for rev in want:
126
 
        todo.update(graph.find_unique_ancestors(rev, have))
 
123
    while pending:
 
124
        processed.update(pending)
 
125
        next_map = get_parent_map(pending)
 
126
        next_pending = set()
 
127
        for item in next_map.iteritems():
 
128
            if item[0] in have:
 
129
                continue
 
130
            todo.add(item[0])
 
131
            next_pending.update(p for p in item[1] if p not in processed)
 
132
        pending = next_pending
127
133
    if NULL_REVISION in todo:
128
134
        todo.remove(NULL_REVISION)
129
135
    return todo
178
184
            except errors.NoSuchId:
179
185
                pass
180
186
            else:
181
 
                if (pie.text_sha1 == ie.text_sha1 and
 
187
                if (pie.text_sha1 == ie.text_sha1 and 
182
188
                    pie.kind == ie.kind and
183
189
                    pie.symlink_target == ie.symlink_target):
184
190
                    return pie
223
229
            tree.inventory[parent[0]].kind == "directory"):
224
230
            # Removal
225
231
            new_trees[posixpath.dirname(path[0])] = parent[0]
226
 
 
 
232
    
227
233
    # Fetch contents of the blobs that were changed
228
234
    for (path, ie), chunks in tree.iter_files_bytes(
229
235
        [(ie.file_id, (path, ie)) for (path, ie) in new_blobs]):
286
292
 
287
293
    def __init__(self, repository, mapping=None):
288
294
        self.repository = repository
289
 
        self._map_updated = False
290
 
        self._locked = None
291
295
        if mapping is None:
292
296
            self.mapping = default_mapping
293
297
        else:
300
304
        self.tree_cache = LRUTreeCache(self.repository)
301
305
 
302
306
    def _update_sha_map(self, stop_revision=None):
303
 
        if not self.is_locked():
304
 
            raise AssertionError()
305
 
        if self._map_updated:
306
 
            return
307
307
        graph = self.repository.get_graph()
308
308
        if stop_revision is None:
309
309
            heads = graph.heads(self.repository.all_revision_ids())
321
321
            missing_revids.remove(NULL_REVISION)
322
322
        missing_revids = self.repository.has_revisions(missing_revids)
323
323
        if not missing_revids:
324
 
            self._map_updated = True
325
324
            return
326
325
        self.start_write_group()
327
326
        try:
333
332
                    self._update_sha_map_revision(revid)
334
333
            finally:
335
334
                pb.finished()
336
 
            self._map_updated = True
337
335
        except:
338
336
            self.abort_write_group()
339
337
            raise
401
399
        if roundtrip and self.mapping.BZR_FILE_IDS_FILE is not None:
402
400
            b = self._create_fileid_map_blob(tree.inventory)
403
401
            if b is not None:
404
 
                root_tree[self.mapping.BZR_FILE_IDS_FILE] = (
405
 
                    (stat.S_IFREG | 0644), b.id)
 
402
                root_tree[self.mapping.BZR_FILE_IDS_FILE] = ((stat.S_IFREG | 0644), b.id)
406
403
                yield self.mapping.BZR_FILE_IDS_FILE, b, None
407
404
        yield "", root_tree, root_ie
408
405
        if roundtrip:
409
 
            if getattr(StrictTestament3, "from_revision_tree", None):
410
 
                testament3 = StrictTestament3(rev, tree)
411
 
            else: # bzr < 2.4
412
 
                testament3 = StrictTestament3(rev, tree.inventory)
 
406
            testament3 = StrictTestament3(rev, tree.inventory)
413
407
            verifiers = { "testament3-sha1": testament3.as_sha1() }
414
408
        else:
415
409
            verifiers = {}
434
428
        for path, obj, ie in self._revision_to_objects(rev, tree,
435
429
            roundtrip=True):
436
430
            if isinstance(obj, Commit):
437
 
                if getattr(StrictTestament3, "from_revision_tree", None):
438
 
                    testament3 = StrictTestament3(rev, tree)
439
 
                else: # bzr < 2.4
440
 
                    testament3 = StrictTestament3(rev, tree.inventory)
 
431
                testament3 = StrictTestament3(rev, tree.inventory)
441
432
                ie = { "testament3-sha1": testament3.as_sha1() }
442
433
            updater.add_object(obj, ie, path)
443
434
        commit_obj = updater.finish()
496
487
            self.mapping.BZR_DUMMY_FILE)
497
488
        if (inv.root.file_id == fileid and
498
489
            self.mapping.BZR_FILE_IDS_FILE is not None):
499
 
            if tree is None:
500
 
                tree = Tree()
501
490
            b = self._create_fileid_map_blob(inv)
502
491
            # If this is the root tree, add the file ids
503
 
            tree[self.mapping.BZR_FILE_IDS_FILE] = (
504
 
                (stat.S_IFREG | 0644), b.id)
505
 
        if tree is not None:
506
 
            _check_expected_sha(expected_sha, tree)
 
492
            tree[self.mapping.BZR_FILE_IDS_FILE] = ((stat.S_IFREG | 0644), b.id)
 
493
        _check_expected_sha(expected_sha, tree)
507
494
        return tree
508
495
 
509
496
    def get_parents(self, sha):
516
503
 
517
504
    def _lookup_revision_sha1(self, revid):
518
505
        """Return the SHA1 matching a Bazaar revision."""
 
506
        from dulwich.protocol import ZERO_SHA
519
507
        if revid == NULL_REVISION:
520
508
            return ZERO_SHA
521
509
        try:
542
530
    def __contains__(self, sha):
543
531
        # See if sha is in map
544
532
        try:
545
 
            for (type, type_data) in self.lookup_git_sha(sha):
546
 
                if type == "commit":
547
 
                    if self.repository.has_revision(type_data[0]):
548
 
                        return True
549
 
                elif type == "blob":
550
 
                    if self.repository.texts.has_key(type_data):
551
 
                        return True
552
 
                elif type == "tree":
553
 
                    if self.repository.has_revision(type_data[1]):
554
 
                        return True
555
 
                else:
556
 
                    raise AssertionError("Unknown object type '%s'" % type)
 
533
            (type, type_data) = self.lookup_git_sha(sha)
 
534
            if type == "commit":
 
535
                return self.repository.has_revision(type_data[0])
 
536
            elif type == "blob":
 
537
                return self.repository.texts.has_key(type_data)
 
538
            elif type == "tree":
 
539
                return self.repository.has_revision(type_data[1])
557
540
            else:
558
 
                return False
 
541
                raise AssertionError("Unknown object type '%s'" % type)
559
542
        except KeyError:
560
543
            return False
561
544
 
562
 
    def lock_read(self):
563
 
        self._locked = 'r'
564
 
        self._map_updated = False
565
 
        self.repository.lock_read()
566
 
        return LogicalLockResult(self.unlock)
567
 
 
568
 
    def lock_write(self):
569
 
        self._locked = 'r'
570
 
        self._map_updated = False
571
 
        self.repository.lock_write()
572
 
        return LogicalLockResult(self.unlock)
573
 
 
574
 
    def is_locked(self):
575
 
        return (self._locked is not None)
576
 
 
577
 
    def unlock(self):
578
 
        self._locked = None
579
 
        self._map_updated = False
580
 
        self.repository.unlock()
581
 
 
582
 
    def lookup_git_shas(self, shas):
 
545
    def lookup_git_shas(self, shas, update_map=True):
 
546
        from dulwich.protocol import ZERO_SHA
583
547
        ret = {}
584
548
        for sha in shas:
585
549
            if sha == ZERO_SHA:
586
 
                ret[sha] = [("commit", (NULL_REVISION, None, {}))]
 
550
                ret[sha] = ("commit", (NULL_REVISION, None, {}))
587
551
                continue
588
552
            try:
589
 
                ret[sha] = list(self._cache.idmap.lookup_git_sha(sha))
 
553
                ret[sha] = self._cache.idmap.lookup_git_sha(sha)
590
554
            except KeyError:
591
 
                # if not, see if there are any unconverted revisions and
592
 
                # add them to the map, search for sha in map again
593
 
                self._update_sha_map()
594
 
                try:
595
 
                    ret[sha] = list(self._cache.idmap.lookup_git_sha(sha))
596
 
                except KeyError:
597
 
                    pass
 
555
                if update_map:
 
556
                    # if not, see if there are any unconverted revisions and add
 
557
                    # them to the map, search for sha in map again
 
558
                    self._update_sha_map()
 
559
                    update_map = False
 
560
                    try:
 
561
                        ret[sha] = self._cache.idmap.lookup_git_sha(sha)
 
562
                    except KeyError:
 
563
                        pass
598
564
        return ret
599
565
 
600
 
    def lookup_git_sha(self, sha):
601
 
        return self.lookup_git_shas([sha])[sha]
 
566
    def lookup_git_sha(self, sha, update_map=True):
 
567
        return self.lookup_git_shas([sha], update_map=update_map)[sha]
602
568
 
603
569
    def __getitem__(self, sha):
604
570
        if self._cache.content_cache is not None:
606
572
                return self._cache.content_cache[sha]
607
573
            except KeyError:
608
574
                pass
609
 
        for (kind, type_data) in self.lookup_git_sha(sha):
610
 
            # convert object to git object
611
 
            if kind == "commit":
612
 
                (revid, tree_sha, verifiers) = type_data
613
 
                try:
614
 
                    rev = self.repository.get_revision(revid)
615
 
                except errors.NoSuchRevision:
616
 
                    trace.mutter('entry for %s %s in shamap: %r, but not '
617
 
                                 'found in repository', kind, sha, type_data)
618
 
                    raise KeyError(sha)
619
 
                commit = self._reconstruct_commit(rev, tree_sha,
620
 
                    roundtrip=True, verifiers=verifiers)
621
 
                _check_expected_sha(sha, commit)
622
 
                return commit
623
 
            elif kind == "blob":
624
 
                (fileid, revision) = type_data
625
 
                blobs = self._reconstruct_blobs([(fileid, revision, sha)])
626
 
                return blobs.next()
627
 
            elif kind == "tree":
628
 
                (fileid, revid) = type_data
629
 
                try:
630
 
                    tree = self.tree_cache.revision_tree(revid)
631
 
                    rev = self.repository.get_revision(revid)
632
 
                except errors.NoSuchRevision:
633
 
                    trace.mutter('entry for %s %s in shamap: %r, but not found in repository', kind, sha, type_data)
634
 
                    raise KeyError(sha)
635
 
                unusual_modes = extract_unusual_modes(rev)
636
 
                try:
637
 
                    return self._reconstruct_tree(fileid, revid,
638
 
                        tree.inventory, unusual_modes, expected_sha=sha)
639
 
                except errors.NoSuchRevision:
640
 
                    raise KeyError(sha)
641
 
            else:
642
 
                raise AssertionError("Unknown object type '%s'" % kind)
 
575
        (type, type_data) = self.lookup_git_sha(sha)
 
576
        # convert object to git object
 
577
        if type == "commit":
 
578
            (revid, tree_sha, verifiers) = type_data
 
579
            try:
 
580
                rev = self.repository.get_revision(revid)
 
581
            except errors.NoSuchRevision:
 
582
                trace.mutter('entry for %s %s in shamap: %r, but not found in '
 
583
                             'repository', type, sha, type_data)
 
584
                raise KeyError(sha)
 
585
            commit = self._reconstruct_commit(rev, tree_sha, roundtrip=True,
 
586
                verifiers=verifiers)
 
587
            _check_expected_sha(sha, commit)
 
588
            return commit
 
589
        elif type == "blob":
 
590
            (fileid, revision) = type_data
 
591
            return self._reconstruct_blobs([(fileid, revision, sha)]).next()
 
592
        elif type == "tree":
 
593
            (fileid, revid) = type_data
 
594
            try:
 
595
                tree = self.tree_cache.revision_tree(revid)
 
596
                rev = self.repository.get_revision(revid)
 
597
            except errors.NoSuchRevision:
 
598
                trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
 
599
                raise KeyError(sha)
 
600
            unusual_modes = extract_unusual_modes(rev)
 
601
            try:
 
602
                return self._reconstruct_tree(fileid, revid, tree.inventory,
 
603
                    unusual_modes, expected_sha=sha)
 
604
            except errors.NoSuchRevision:
 
605
                raise KeyError(sha)
643
606
        else:
644
 
            raise KeyError(sha)
 
607
            raise AssertionError("Unknown object type '%s'" % type)
645
608
 
646
609
    def generate_lossy_pack_contents(self, have, want, progress=None,
647
610
            get_tagged=None):
659
622
        ret = self.lookup_git_shas(have + want)
660
623
        for commit_sha in have:
661
624
            try:
662
 
                for (type, type_data) in ret[commit_sha]:
663
 
                    assert type == "commit"
664
 
                    processed.add(type_data[0])
 
625
                (type, (revid, tree_sha)) = ret[commit_sha]
665
626
            except KeyError:
666
627
                pass
 
628
            else:
 
629
                assert type == "commit"
 
630
                processed.add(revid)
667
631
        pending = set()
668
632
        for commit_sha in want:
669
633
            if commit_sha in have:
670
634
                continue
671
635
            try:
672
 
                for (type, type_data) in ret[commit_sha]:
673
 
                    assert type == "commit"
674
 
                    pending.add(type_data[0])
 
636
                (type, (revid, tree_sha)) = ret[commit_sha]
675
637
            except KeyError:
676
638
                pass
 
639
            else:
 
640
                assert type == "commit"
 
641
                pending.add(revid)
677
642
 
678
 
        graph = self.repository.get_graph()
679
 
        todo = _find_missing_bzr_revids(graph, pending, processed)
 
643
        todo = _find_missing_bzr_revids(self.repository.get_parent_map, 
 
644
                                        pending, processed)
680
645
        trace.mutter('sending revisions %r', todo)
681
646
        ret = []
682
647
        pb = ui.ui_factory.nested_progress_bar()
683
648
        try:
684
649
            for i, revid in enumerate(todo):
685
650
                pb.update("generating git objects", i, len(todo))
686
 
                try:
687
 
                    rev = self.repository.get_revision(revid)
688
 
                except errors.NoSuchRevision:
689
 
                    continue
 
651
                rev = self.repository.get_revision(revid)
690
652
                tree = self.tree_cache.revision_tree(revid)
691
653
                for path, obj, ie in self._revision_to_objects(rev, tree,
692
654
                    roundtrip=not lossy):