/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
0.200.252 by Jelmer Vernooij
Clarify history, copyright.
1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
0.200.228 by Jelmer Vernooij
Split out map.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Map from Git sha's to Bazaar objects."""
18
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
19
from dulwich.objects import (
20
    Blob,
21
    Tree,
22
    )
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
23
from dulwich.object_store import (
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
24
    BaseObjectStore,
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
25
    ObjectStoreIterator,
26
    )
0.200.249 by Jelmer Vernooij
Implement Tree.
27
import stat
28
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
29
from bzrlib import (
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
30
    debug,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
31
    errors,
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
32
    ui,
33
    )
0.200.228 by Jelmer Vernooij
Split out map.
34
0.200.467 by Jelmer Vernooij
Raise proper errors for ghosts.
35
from bzrlib.plugins.git.errors import (
36
    GhostRevision,
37
    )
38
0.200.229 by Jelmer Vernooij
More work on converter.
39
from bzrlib.plugins.git.mapping import (
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
40
    default_mapping,
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
41
    directory_to_tree,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
42
    mapping_registry,
0.200.229 by Jelmer Vernooij
More work on converter.
43
    revision_to_commit,
44
    )
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
45
from bzrlib.plugins.git.shamap import (
46
    SqliteGitShaMap,
0.200.231 by Jelmer Vernooij
Partially fix pull.
47
    )
48
0.200.228 by Jelmer Vernooij
Split out map.
49
0.200.452 by Jelmer Vernooij
Rename converter -> object_store, provide utility function for getting ObjectStore's.
50
def get_object_store(repo, mapping=None):
51
    git = getattr(repo, "_git", None)
52
    if git is not None:
53
        return git.object_store
54
    return BazaarObjectStore(repo, mapping)
55
56
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
57
class BazaarObjectStore(BaseObjectStore):
0.200.320 by Jelmer Vernooij
Handle lightweight checkouts.
58
    """A Git-style object store backed onto a Bazaar repository."""
0.200.228 by Jelmer Vernooij
Split out map.
59
60
    def __init__(self, repository, mapping=None):
61
        self.repository = repository
62
        if mapping is None:
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
63
            self.mapping = default_mapping
0.200.228 by Jelmer Vernooij
Split out map.
64
        else:
65
            self.mapping = mapping
0.200.365 by Jelmer Vernooij
Share sha map cache connections inside threads.
66
        self._idmap = SqliteGitShaMap.from_repository(repository)
0.200.228 by Jelmer Vernooij
Split out map.
67
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
68
    def _update_sha_map(self, stop_revision=None):
69
        if stop_revision is None:
70
            all_revids = self.repository.all_revision_ids()
71
        else:
72
            all_revids = self.repository.get_ancestry(stop_revision)
73
            first = all_revids.pop(0) # Pop leading None
74
            assert first is None
0.200.231 by Jelmer Vernooij
Partially fix pull.
75
        graph = self.repository.get_graph()
0.200.230 by Jelmer Vernooij
Implement sha cache.
76
        present_revids = set(self._idmap.revids())
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
77
        missing_revids = [revid for revid in graph.iter_topo_order(all_revids) if revid not in present_revids]
0.200.231 by Jelmer Vernooij
Partially fix pull.
78
        pb = ui.ui_factory.nested_progress_bar()
79
        try:
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
80
            for i, revid in enumerate(missing_revids):
81
                pb.update("updating git map", i, len(missing_revids))
0.200.231 by Jelmer Vernooij
Partially fix pull.
82
                self._update_sha_map_revision(revid)
83
        finally:
0.200.232 by Jelmer Vernooij
Fix pull from remote branches.
84
            self._idmap.commit()
0.200.231 by Jelmer Vernooij
Partially fix pull.
85
            pb.finished()
0.200.229 by Jelmer Vernooij
More work on converter.
86
0.200.422 by Jelmer Vernooij
'bzr git-object' without arguments now prints the available git objects.
87
    def __iter__(self):
88
        self._update_sha_map()
89
        return iter(self._idmap.sha1s())
90
0.200.229 by Jelmer Vernooij
More work on converter.
91
    def _update_sha_map_revision(self, revid):
92
        inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
93
        tree_sha = self._get_ie_sha1(inv.root, inv)
0.200.229 by Jelmer Vernooij
More work on converter.
94
        rev = self.repository.get_revision(revid)
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
95
        commit_obj = revision_to_commit(rev, tree_sha,
96
            self._idmap._parent_lookup)
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
97
        try:
98
            foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
99
        except errors.InvalidRevisionId:
100
            pass
101
        else:
102
            if foreign_revid != commit_obj.id:
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
103
                if not "fix-shamap" in debug.debug_flags:
104
                    raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
105
        self._idmap.add_entry(commit_obj.id, "commit", (revid, tree_sha))
0.200.229 by Jelmer Vernooij
More work on converter.
106
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
107
    def _check_expected_sha(self, expected_sha, object):
108
        if expected_sha is None:
109
            return
110
        if expected_sha != object.id:
111
            raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
112
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
113
    def _get_ie_object(self, entry, inv):  
114
        if entry.kind == "directory":
115
            return self._get_tree(entry.file_id, inv.revision_id, inv=inv)
116
        else:
117
            return self._get_blob(entry.file_id, entry.revision)
118
119
    def _get_ie_object_or_sha1(self, entry, inv):
120
        if entry.kind == "directory":
121
            try:
122
                return self._idmap.lookup_tree(entry.file_id, inv.revision_id), None
123
            except KeyError:
124
                ret = self._get_ie_object(entry, inv)
125
                self._idmap.add_entry(ret.id, "tree", (entry.file_id, inv.revision_id))
126
                return ret.id, ret
127
        else:
128
            try:
129
                return self._idmap.lookup_blob(entry.file_id, entry.revision), None
130
            except KeyError:
131
                ret = self._get_ie_object(entry, inv)
132
                self._idmap.add_entry(ret.id, "blob", (entry.file_id, entry.revision))
133
                return ret.id, ret
134
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
135
    def _get_ie_sha1(self, entry, inv):
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
136
        return self._get_ie_object_or_sha1(entry, inv)[0]
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
137
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
138
    def _get_blob(self, fileid, revision, expected_sha=None):
0.200.236 by Jelmer Vernooij
require bzr 1.13.
139
        """Return a Git Blob object from a fileid and revision stored in bzr.
140
        
141
        :param fileid: File id of the text
142
        :param revision: Revision of the text
143
        """
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
144
        text = self.repository.texts.get_record_stream([(fileid, revision)],
145
            "unordered", True).next().get_bytes_as("fulltext")
0.200.229 by Jelmer Vernooij
More work on converter.
146
        blob = Blob()
147
        blob._text = text
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
148
        self._check_expected_sha(expected_sha, blob)
0.200.229 by Jelmer Vernooij
More work on converter.
149
        return blob
150
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
151
    def _get_tree(self, fileid, revid, inv=None, expected_sha=None):
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
152
        """Return a Git Tree object from a file id and a revision stored in bzr.
0.200.249 by Jelmer Vernooij
Implement Tree.
153
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
154
        :param fileid: fileid in the tree.
0.200.249 by Jelmer Vernooij
Implement Tree.
155
        :param revision: Revision of the tree.
156
        """
157
        if inv is None:
158
            inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
159
        tree = directory_to_tree(inv[fileid], lambda ie: self._get_ie_sha1(ie, inv))
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
160
        self._check_expected_sha(expected_sha, tree)
0.200.249 by Jelmer Vernooij
Implement Tree.
161
        return tree
0.200.229 by Jelmer Vernooij
More work on converter.
162
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
163
    def _get_commit(self, revid, tree_sha, expected_sha=None):
0.200.229 by Jelmer Vernooij
More work on converter.
164
        rev = self.repository.get_revision(revid)
0.200.467 by Jelmer Vernooij
Raise proper errors for ghosts.
165
        try:
166
            commit = revision_to_commit(rev, tree_sha, self._lookup_revision_sha1)
167
        except errors.NoSuchRevision, e:
168
            raise GhostRevision(e.branch, e.revision)
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
169
        self._check_expected_sha(expected_sha, commit)
170
        return commit
0.200.228 by Jelmer Vernooij
Split out map.
171
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
172
    def get_parents(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
173
        """Retrieve the parents of a Git commit by SHA1.
174
175
        :param sha: SHA1 of the commit
176
        :raises: KeyError, NotCommitError
177
        """
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
178
        return self[sha].parents
179
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
180
    def _lookup_revision_sha1(self, revid):
0.200.449 by Jelmer Vernooij
Use BazaarObjectStore to find matching SHA1s for bzr revisions.
181
        """Return the SHA1 matching a Bazaar revision."""
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
182
        try:
183
            return self._idmap._parent_lookup(revid)
184
        except KeyError:
185
            inv = self.repository.get_inventory(revid)
186
            tree_sha = self._get_ie_sha1(inv.root, inv)
187
            ret = self._get_commit(revid, tree_sha).id
188
            self._idmap.add_entry(ret, "commit", (revid, tree_sha))
189
            return ret
190
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
191
    def get_raw(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
192
        """Get the raw representation of a Git object by SHA1.
193
194
        :param sha: SHA1 of the git object
195
        """
0.200.421 by Jelmer Vernooij
Use public method to access raw git object text.
196
        return self[sha].as_raw_string()
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
197
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
198
    def __contains__(self, sha):
199
        # See if sha is in map
200
        try:
201
            self._lookup_git_sha(sha)
202
        except KeyError:
203
            return False
204
        else:
205
            return True
206
207
    def _lookup_git_sha(self, sha):
208
        # See if sha is in map
209
        try:
210
            return self._idmap.lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
211
        except KeyError:
212
            # if not, see if there are any unconverted revisions and add them 
213
            # to the map, search for sha in map again
214
            self._update_sha_map()
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
215
            return self._idmap.lookup_git_sha(sha)
216
217
    def __getitem__(self, sha):
218
        (type, type_data) = self._lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
219
        # convert object to git object
0.200.229 by Jelmer Vernooij
More work on converter.
220
        if type == "commit":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
221
            return self._get_commit(type_data[0], type_data[1], 
222
                                    expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
223
        elif type == "blob":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
224
            return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
225
        elif type == "tree":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
226
            return self._get_tree(type_data[0], type_data[1], expected_sha=sha)
0.200.228 by Jelmer Vernooij
Split out map.
227
        else:
228
            raise AssertionError("Unknown object type '%s'" % type)