/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
0.200.252 by Jelmer Vernooij
Clarify history, copyright.
1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
0.200.228 by Jelmer Vernooij
Split out map.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Map from Git sha's to Bazaar objects."""
18
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
19
from dulwich.objects import (
20
    Blob,
21
    Tree,
22
    )
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
23
from dulwich.object_store import (
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
24
    BaseObjectStore,
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
25
    ObjectStoreIterator,
26
    )
0.200.249 by Jelmer Vernooij
Implement Tree.
27
import stat
28
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
29
from bzrlib import (
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
30
    debug,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
31
    errors,
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
32
    ui,
33
    )
0.200.228 by Jelmer Vernooij
Split out map.
34
0.200.467 by Jelmer Vernooij
Raise proper errors for ghosts.
35
from bzrlib.plugins.git.errors import (
36
    GhostRevision,
37
    )
38
0.200.229 by Jelmer Vernooij
More work on converter.
39
from bzrlib.plugins.git.mapping import (
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
40
    default_mapping,
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
41
    directory_to_tree,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
42
    mapping_registry,
0.200.229 by Jelmer Vernooij
More work on converter.
43
    revision_to_commit,
44
    )
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
45
from bzrlib.plugins.git.shamap import (
46
    SqliteGitShaMap,
0.200.476 by Jelmer Vernooij
Fix Tdb backend, use tdb if possible by default.
47
    TdbGitShaMap,
0.200.231 by Jelmer Vernooij
Partially fix pull.
48
    )
49
0.200.228 by Jelmer Vernooij
Split out map.
50
0.200.452 by Jelmer Vernooij
Rename converter -> object_store, provide utility function for getting ObjectStore's.
51
def get_object_store(repo, mapping=None):
52
    git = getattr(repo, "_git", None)
53
    if git is not None:
54
        return git.object_store
55
    return BazaarObjectStore(repo, mapping)
56
57
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
58
class BazaarObjectStore(BaseObjectStore):
0.200.320 by Jelmer Vernooij
Handle lightweight checkouts.
59
    """A Git-style object store backed onto a Bazaar repository."""
0.200.228 by Jelmer Vernooij
Split out map.
60
61
    def __init__(self, repository, mapping=None):
62
        self.repository = repository
63
        if mapping is None:
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
64
            self.mapping = default_mapping
0.200.228 by Jelmer Vernooij
Split out map.
65
        else:
66
            self.mapping = mapping
0.200.476 by Jelmer Vernooij
Fix Tdb backend, use tdb if possible by default.
67
        try:
68
            self._idmap = TdbGitShaMap.from_repository(repository)
69
        except ImportError:
70
            self._idmap = SqliteGitShaMap.from_repository(repository)
0.200.228 by Jelmer Vernooij
Split out map.
71
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
72
    def _update_sha_map(self, stop_revision=None):
73
        if stop_revision is None:
74
            all_revids = self.repository.all_revision_ids()
75
        else:
76
            all_revids = self.repository.get_ancestry(stop_revision)
77
            first = all_revids.pop(0) # Pop leading None
78
            assert first is None
0.200.231 by Jelmer Vernooij
Partially fix pull.
79
        graph = self.repository.get_graph()
0.200.230 by Jelmer Vernooij
Implement sha cache.
80
        present_revids = set(self._idmap.revids())
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
81
        missing_revids = [revid for revid in graph.iter_topo_order(all_revids) if revid not in present_revids]
0.200.231 by Jelmer Vernooij
Partially fix pull.
82
        pb = ui.ui_factory.nested_progress_bar()
83
        try:
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
84
            for i, revid in enumerate(missing_revids):
85
                pb.update("updating git map", i, len(missing_revids))
0.200.231 by Jelmer Vernooij
Partially fix pull.
86
                self._update_sha_map_revision(revid)
87
        finally:
0.200.232 by Jelmer Vernooij
Fix pull from remote branches.
88
            self._idmap.commit()
0.200.231 by Jelmer Vernooij
Partially fix pull.
89
            pb.finished()
0.200.229 by Jelmer Vernooij
More work on converter.
90
0.200.422 by Jelmer Vernooij
'bzr git-object' without arguments now prints the available git objects.
91
    def __iter__(self):
92
        self._update_sha_map()
93
        return iter(self._idmap.sha1s())
94
0.200.229 by Jelmer Vernooij
More work on converter.
95
    def _update_sha_map_revision(self, revid):
96
        inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
97
        tree_sha = self._get_ie_sha1(inv.root, inv)
0.200.229 by Jelmer Vernooij
More work on converter.
98
        rev = self.repository.get_revision(revid)
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
99
        commit_obj = revision_to_commit(rev, tree_sha,
100
            self._idmap._parent_lookup)
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
101
        try:
102
            foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
103
        except errors.InvalidRevisionId:
104
            pass
105
        else:
106
            if foreign_revid != commit_obj.id:
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
107
                if not "fix-shamap" in debug.debug_flags:
108
                    raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
109
        self._idmap.add_entry(commit_obj.id, "commit", (revid, tree_sha))
0.200.229 by Jelmer Vernooij
More work on converter.
110
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
111
    def _check_expected_sha(self, expected_sha, object):
112
        if expected_sha is None:
113
            return
114
        if expected_sha != object.id:
115
            raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
116
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
117
    def _get_ie_object(self, entry, inv):  
118
        if entry.kind == "directory":
119
            return self._get_tree(entry.file_id, inv.revision_id, inv=inv)
120
        else:
121
            return self._get_blob(entry.file_id, entry.revision)
122
123
    def _get_ie_object_or_sha1(self, entry, inv):
124
        if entry.kind == "directory":
125
            try:
126
                return self._idmap.lookup_tree(entry.file_id, inv.revision_id), None
127
            except KeyError:
128
                ret = self._get_ie_object(entry, inv)
129
                self._idmap.add_entry(ret.id, "tree", (entry.file_id, inv.revision_id))
130
                return ret.id, ret
131
        else:
132
            try:
133
                return self._idmap.lookup_blob(entry.file_id, entry.revision), None
134
            except KeyError:
135
                ret = self._get_ie_object(entry, inv)
136
                self._idmap.add_entry(ret.id, "blob", (entry.file_id, entry.revision))
137
                return ret.id, ret
138
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
139
    def _get_ie_sha1(self, entry, inv):
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
140
        return self._get_ie_object_or_sha1(entry, inv)[0]
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
141
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
142
    def _get_blob(self, fileid, revision, expected_sha=None):
0.200.236 by Jelmer Vernooij
require bzr 1.13.
143
        """Return a Git Blob object from a fileid and revision stored in bzr.
144
        
145
        :param fileid: File id of the text
146
        :param revision: Revision of the text
147
        """
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
148
        text = self.repository.texts.get_record_stream([(fileid, revision)],
149
            "unordered", True).next().get_bytes_as("fulltext")
0.200.229 by Jelmer Vernooij
More work on converter.
150
        blob = Blob()
151
        blob._text = text
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
152
        self._check_expected_sha(expected_sha, blob)
0.200.229 by Jelmer Vernooij
More work on converter.
153
        return blob
154
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
155
    def _get_tree(self, fileid, revid, inv=None, expected_sha=None):
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
156
        """Return a Git Tree object from a file id and a revision stored in bzr.
0.200.249 by Jelmer Vernooij
Implement Tree.
157
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
158
        :param fileid: fileid in the tree.
0.200.249 by Jelmer Vernooij
Implement Tree.
159
        :param revision: Revision of the tree.
160
        """
161
        if inv is None:
162
            inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
163
        tree = directory_to_tree(inv[fileid], lambda ie: self._get_ie_sha1(ie, inv))
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
164
        self._check_expected_sha(expected_sha, tree)
0.200.249 by Jelmer Vernooij
Implement Tree.
165
        return tree
0.200.229 by Jelmer Vernooij
More work on converter.
166
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
167
    def _get_commit(self, revid, tree_sha, expected_sha=None):
0.200.229 by Jelmer Vernooij
More work on converter.
168
        rev = self.repository.get_revision(revid)
0.200.467 by Jelmer Vernooij
Raise proper errors for ghosts.
169
        try:
170
            commit = revision_to_commit(rev, tree_sha, self._lookup_revision_sha1)
171
        except errors.NoSuchRevision, e:
172
            raise GhostRevision(e.branch, e.revision)
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
173
        self._check_expected_sha(expected_sha, commit)
174
        return commit
0.200.228 by Jelmer Vernooij
Split out map.
175
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
176
    def get_parents(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
177
        """Retrieve the parents of a Git commit by SHA1.
178
179
        :param sha: SHA1 of the commit
180
        :raises: KeyError, NotCommitError
181
        """
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
182
        return self[sha].parents
183
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
184
    def _lookup_revision_sha1(self, revid):
0.200.449 by Jelmer Vernooij
Use BazaarObjectStore to find matching SHA1s for bzr revisions.
185
        """Return the SHA1 matching a Bazaar revision."""
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
186
        try:
187
            return self._idmap._parent_lookup(revid)
188
        except KeyError:
189
            inv = self.repository.get_inventory(revid)
190
            tree_sha = self._get_ie_sha1(inv.root, inv)
191
            ret = self._get_commit(revid, tree_sha).id
192
            self._idmap.add_entry(ret, "commit", (revid, tree_sha))
193
            return ret
194
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
195
    def get_raw(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
196
        """Get the raw representation of a Git object by SHA1.
197
198
        :param sha: SHA1 of the git object
199
        """
0.200.421 by Jelmer Vernooij
Use public method to access raw git object text.
200
        return self[sha].as_raw_string()
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
201
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
202
    def __contains__(self, sha):
203
        # See if sha is in map
204
        try:
205
            self._lookup_git_sha(sha)
206
        except KeyError:
207
            return False
208
        else:
209
            return True
210
211
    def _lookup_git_sha(self, sha):
212
        # See if sha is in map
213
        try:
214
            return self._idmap.lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
215
        except KeyError:
216
            # if not, see if there are any unconverted revisions and add them 
217
            # to the map, search for sha in map again
218
            self._update_sha_map()
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
219
            return self._idmap.lookup_git_sha(sha)
220
221
    def __getitem__(self, sha):
222
        (type, type_data) = self._lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
223
        # convert object to git object
0.200.229 by Jelmer Vernooij
More work on converter.
224
        if type == "commit":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
225
            return self._get_commit(type_data[0], type_data[1], 
226
                                    expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
227
        elif type == "blob":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
228
            return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
229
        elif type == "tree":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
230
            return self._get_tree(type_data[0], type_data[1], expected_sha=sha)
0.200.228 by Jelmer Vernooij
Split out map.
231
        else:
232
            raise AssertionError("Unknown object type '%s'" % type)