/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
0.200.252 by Jelmer Vernooij
Clarify history, copyright.
1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
0.200.228 by Jelmer Vernooij
Split out map.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Map from Git sha's to Bazaar objects."""
18
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
19
from dulwich.objects import (
20
    Blob,
21
    Tree,
22
    )
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
23
from dulwich.object_store import (
24
    ObjectStoreIterator,
25
    )
0.200.249 by Jelmer Vernooij
Implement Tree.
26
import stat
27
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
28
from bzrlib import (
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
29
    debug,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
30
    errors,
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
31
    ui,
32
    )
0.200.228 by Jelmer Vernooij
Split out map.
33
0.200.229 by Jelmer Vernooij
More work on converter.
34
from bzrlib.plugins.git.mapping import (
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
35
    directory_to_tree,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
36
    mapping_registry,
0.200.229 by Jelmer Vernooij
More work on converter.
37
    revision_to_commit,
38
    )
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
39
from bzrlib.plugins.git.shamap import (
40
    SqliteGitShaMap,
0.200.231 by Jelmer Vernooij
Partially fix pull.
41
    )
42
0.200.228 by Jelmer Vernooij
Split out map.
43
0.200.320 by Jelmer Vernooij
Handle lightweight checkouts.
44
class BazaarObjectStore(object):
45
    """A Git-style object store backed onto a Bazaar repository."""
0.200.228 by Jelmer Vernooij
Split out map.
46
47
    def __init__(self, repository, mapping=None):
48
        self.repository = repository
49
        if mapping is None:
50
            self.mapping = self.repository.get_mapping()
51
        else:
52
            self.mapping = mapping
0.200.365 by Jelmer Vernooij
Share sha map cache connections inside threads.
53
        self._idmap = SqliteGitShaMap.from_repository(repository)
0.200.228 by Jelmer Vernooij
Split out map.
54
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
55
    def iter_shas(self, shas):
56
        return ObjectStoreIterator(self, shas)
57
58
    def _update_sha_map(self, stop_revision=None):
59
        if stop_revision is None:
60
            all_revids = self.repository.all_revision_ids()
61
        else:
62
            all_revids = self.repository.get_ancestry(stop_revision)
63
            first = all_revids.pop(0) # Pop leading None
64
            assert first is None
0.200.231 by Jelmer Vernooij
Partially fix pull.
65
        graph = self.repository.get_graph()
0.200.230 by Jelmer Vernooij
Implement sha cache.
66
        present_revids = set(self._idmap.revids())
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
67
        missing_revids = [revid for revid in graph.iter_topo_order(all_revids) if revid not in present_revids]
0.200.231 by Jelmer Vernooij
Partially fix pull.
68
        pb = ui.ui_factory.nested_progress_bar()
69
        try:
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
70
            for i, revid in enumerate(missing_revids):
71
                pb.update("updating git map", i, len(missing_revids))
0.200.231 by Jelmer Vernooij
Partially fix pull.
72
                self._update_sha_map_revision(revid)
73
        finally:
0.200.232 by Jelmer Vernooij
Fix pull from remote branches.
74
            self._idmap.commit()
0.200.231 by Jelmer Vernooij
Partially fix pull.
75
            pb.finished()
0.200.229 by Jelmer Vernooij
More work on converter.
76
0.200.422 by Jelmer Vernooij
'bzr git-object' without arguments now prints the available git objects.
77
    def __iter__(self):
78
        self._update_sha_map()
79
        return iter(self._idmap.sha1s())
80
0.200.229 by Jelmer Vernooij
More work on converter.
81
    def _update_sha_map_revision(self, revid):
82
        inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
83
        tree_sha = self._get_ie_sha1(inv.root, inv)
0.200.229 by Jelmer Vernooij
More work on converter.
84
        rev = self.repository.get_revision(revid)
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
85
        commit_obj = revision_to_commit(rev, tree_sha,
86
            self._idmap._parent_lookup)
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
87
        try:
88
            foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
89
        except errors.InvalidRevisionId:
90
            pass
91
        else:
92
            if foreign_revid != commit_obj.id:
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
93
                if not "fix-shamap" in debug.debug_flags:
94
                    raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
95
        self._idmap.add_entry(commit_obj.id, "commit", (revid, tree_sha))
0.200.229 by Jelmer Vernooij
More work on converter.
96
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
97
    def _check_expected_sha(self, expected_sha, object):
98
        if expected_sha is None:
99
            return
100
        if expected_sha != object.id:
101
            raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
102
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
103
    def _get_ie_object(self, entry, inv):  
104
        if entry.kind == "directory":
105
            return self._get_tree(entry.file_id, inv.revision_id, inv=inv)
106
        else:
107
            return self._get_blob(entry.file_id, entry.revision)
108
109
    def _get_ie_object_or_sha1(self, entry, inv):
110
        if entry.kind == "directory":
111
            try:
112
                return self._idmap.lookup_tree(entry.file_id, inv.revision_id), None
113
            except KeyError:
114
                ret = self._get_ie_object(entry, inv)
115
                self._idmap.add_entry(ret.id, "tree", (entry.file_id, inv.revision_id))
116
                return ret.id, ret
117
        else:
118
            try:
119
                return self._idmap.lookup_blob(entry.file_id, entry.revision), None
120
            except KeyError:
121
                ret = self._get_ie_object(entry, inv)
122
                self._idmap.add_entry(ret.id, "blob", (entry.file_id, entry.revision))
123
                return ret.id, ret
124
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
125
    def _get_ie_sha1(self, entry, inv):
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
126
        return self._get_ie_object_or_sha1(entry, inv)[0]
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
127
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
128
    def _get_blob(self, fileid, revision, expected_sha=None):
0.200.236 by Jelmer Vernooij
require bzr 1.13.
129
        """Return a Git Blob object from a fileid and revision stored in bzr.
130
        
131
        :param fileid: File id of the text
132
        :param revision: Revision of the text
133
        """
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
134
        text = self.repository.texts.get_record_stream([(fileid, revision)],
135
            "unordered", True).next().get_bytes_as("fulltext")
0.200.229 by Jelmer Vernooij
More work on converter.
136
        blob = Blob()
137
        blob._text = text
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
138
        self._check_expected_sha(expected_sha, blob)
0.200.229 by Jelmer Vernooij
More work on converter.
139
        return blob
140
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
141
    def _get_tree(self, fileid, revid, inv=None, expected_sha=None):
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
142
        """Return a Git Tree object from a file id and a revision stored in bzr.
0.200.249 by Jelmer Vernooij
Implement Tree.
143
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
144
        :param fileid: fileid in the tree.
0.200.249 by Jelmer Vernooij
Implement Tree.
145
        :param revision: Revision of the tree.
146
        """
147
        if inv is None:
148
            inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
149
        tree = directory_to_tree(inv[fileid], lambda ie: self._get_ie_sha1(ie, inv))
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
150
        self._check_expected_sha(expected_sha, tree)
0.200.249 by Jelmer Vernooij
Implement Tree.
151
        return tree
0.200.229 by Jelmer Vernooij
More work on converter.
152
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
153
    def _get_commit(self, revid, tree_sha, expected_sha=None):
0.200.229 by Jelmer Vernooij
More work on converter.
154
        rev = self.repository.get_revision(revid)
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
155
        commit = revision_to_commit(rev, tree_sha, self._lookup_revision_sha1)
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
156
        self._check_expected_sha(expected_sha, commit)
157
        return commit
0.200.228 by Jelmer Vernooij
Split out map.
158
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
159
    def get_parents(self, sha):
160
        return self[sha].parents
161
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
162
    def _lookup_revision_sha1(self, revid):
0.200.449 by Jelmer Vernooij
Use BazaarObjectStore to find matching SHA1s for bzr revisions.
163
        """Return the SHA1 matching a Bazaar revision."""
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
164
        try:
165
            return self._idmap._parent_lookup(revid)
166
        except KeyError:
167
            inv = self.repository.get_inventory(revid)
168
            tree_sha = self._get_ie_sha1(inv.root, inv)
169
            ret = self._get_commit(revid, tree_sha).id
170
            self._idmap.add_entry(ret, "commit", (revid, tree_sha))
171
            return ret
172
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
173
    def get_raw(self, sha):
0.200.421 by Jelmer Vernooij
Use public method to access raw git object text.
174
        return self[sha].as_raw_string()
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
175
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
176
    def __contains__(self, sha):
177
        # See if sha is in map
178
        try:
179
            self._lookup_git_sha(sha)
180
        except KeyError:
181
            return False
182
        else:
183
            return True
184
185
    def _lookup_git_sha(self, sha):
186
        # See if sha is in map
187
        try:
188
            return self._idmap.lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
189
        except KeyError:
190
            # if not, see if there are any unconverted revisions and add them 
191
            # to the map, search for sha in map again
192
            self._update_sha_map()
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
193
            return self._idmap.lookup_git_sha(sha)
194
195
    def __getitem__(self, sha):
196
        (type, type_data) = self._lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
197
        # convert object to git object
0.200.229 by Jelmer Vernooij
More work on converter.
198
        if type == "commit":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
199
            return self._get_commit(type_data[0], type_data[1], 
200
                                    expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
201
        elif type == "blob":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
202
            return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
203
        elif type == "tree":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
204
            return self._get_tree(type_data[0], type_data[1], expected_sha=sha)
0.200.228 by Jelmer Vernooij
Split out map.
205
        else:
206
            raise AssertionError("Unknown object type '%s'" % type)