/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
0.200.252 by Jelmer Vernooij
Clarify history, copyright.
1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
0.200.228 by Jelmer Vernooij
Split out map.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Map from Git sha's to Bazaar objects."""
18
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
19
from dulwich.objects import (
20
    Blob,
21
    Tree,
22
    )
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
23
from dulwich.object_store import (
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
24
    BaseObjectStore,
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
25
    ObjectStoreIterator,
26
    )
0.200.249 by Jelmer Vernooij
Implement Tree.
27
import stat
28
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
29
from bzrlib import (
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
30
    debug,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
31
    errors,
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
32
    ui,
33
    )
0.200.228 by Jelmer Vernooij
Split out map.
34
0.200.229 by Jelmer Vernooij
More work on converter.
35
from bzrlib.plugins.git.mapping import (
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
36
    default_mapping,
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
37
    directory_to_tree,
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
38
    mapping_registry,
0.200.229 by Jelmer Vernooij
More work on converter.
39
    revision_to_commit,
40
    )
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
41
from bzrlib.plugins.git.shamap import (
42
    SqliteGitShaMap,
0.200.231 by Jelmer Vernooij
Partially fix pull.
43
    )
44
0.200.228 by Jelmer Vernooij
Split out map.
45
0.200.452 by Jelmer Vernooij
Rename converter -> object_store, provide utility function for getting ObjectStore's.
46
def get_object_store(repo, mapping=None):
47
    git = getattr(repo, "_git", None)
48
    if git is not None:
49
        return git.object_store
50
    return BazaarObjectStore(repo, mapping)
51
52
0.200.457 by Jelmer Vernooij
Use BaseObjectStore.
53
class BazaarObjectStore(BaseObjectStore):
0.200.320 by Jelmer Vernooij
Handle lightweight checkouts.
54
    """A Git-style object store backed onto a Bazaar repository."""
0.200.228 by Jelmer Vernooij
Split out map.
55
56
    def __init__(self, repository, mapping=None):
57
        self.repository = repository
58
        if mapping is None:
0.200.463 by Jelmer Vernooij
Support remote dpush (except for references).
59
            self.mapping = default_mapping
0.200.228 by Jelmer Vernooij
Split out map.
60
        else:
61
            self.mapping = mapping
0.200.365 by Jelmer Vernooij
Share sha map cache connections inside threads.
62
        self._idmap = SqliteGitShaMap.from_repository(repository)
0.200.228 by Jelmer Vernooij
Split out map.
63
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
64
    def _update_sha_map(self, stop_revision=None):
65
        if stop_revision is None:
66
            all_revids = self.repository.all_revision_ids()
67
        else:
68
            all_revids = self.repository.get_ancestry(stop_revision)
69
            first = all_revids.pop(0) # Pop leading None
70
            assert first is None
0.200.231 by Jelmer Vernooij
Partially fix pull.
71
        graph = self.repository.get_graph()
0.200.230 by Jelmer Vernooij
Implement sha cache.
72
        present_revids = set(self._idmap.revids())
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
73
        missing_revids = [revid for revid in graph.iter_topo_order(all_revids) if revid not in present_revids]
0.200.231 by Jelmer Vernooij
Partially fix pull.
74
        pb = ui.ui_factory.nested_progress_bar()
75
        try:
0.200.347 by Jelmer Vernooij
Simplify converter a bit.
76
            for i, revid in enumerate(missing_revids):
77
                pb.update("updating git map", i, len(missing_revids))
0.200.231 by Jelmer Vernooij
Partially fix pull.
78
                self._update_sha_map_revision(revid)
79
        finally:
0.200.232 by Jelmer Vernooij
Fix pull from remote branches.
80
            self._idmap.commit()
0.200.231 by Jelmer Vernooij
Partially fix pull.
81
            pb.finished()
0.200.229 by Jelmer Vernooij
More work on converter.
82
0.200.422 by Jelmer Vernooij
'bzr git-object' without arguments now prints the available git objects.
83
    def __iter__(self):
84
        self._update_sha_map()
85
        return iter(self._idmap.sha1s())
86
0.200.229 by Jelmer Vernooij
More work on converter.
87
    def _update_sha_map_revision(self, revid):
88
        inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
89
        tree_sha = self._get_ie_sha1(inv.root, inv)
0.200.229 by Jelmer Vernooij
More work on converter.
90
        rev = self.repository.get_revision(revid)
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
91
        commit_obj = revision_to_commit(rev, tree_sha,
92
            self._idmap._parent_lookup)
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
93
        try:
94
            foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
95
        except errors.InvalidRevisionId:
96
            pass
97
        else:
98
            if foreign_revid != commit_obj.id:
0.200.440 by Jelmer Vernooij
Remove silly mapping of timezones; dulwich uses offsets now as well.
99
                if not "fix-shamap" in debug.debug_flags:
100
                    raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
0.231.1 by Jelmer Vernooij
Check that regenerated objects have the expected sha1.
101
        self._idmap.add_entry(commit_obj.id, "commit", (revid, tree_sha))
0.200.229 by Jelmer Vernooij
More work on converter.
102
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
103
    def _check_expected_sha(self, expected_sha, object):
104
        if expected_sha is None:
105
            return
106
        if expected_sha != object.id:
107
            raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
108
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
109
    def _get_ie_object(self, entry, inv):  
110
        if entry.kind == "directory":
111
            return self._get_tree(entry.file_id, inv.revision_id, inv=inv)
112
        else:
113
            return self._get_blob(entry.file_id, entry.revision)
114
115
    def _get_ie_object_or_sha1(self, entry, inv):
116
        if entry.kind == "directory":
117
            try:
118
                return self._idmap.lookup_tree(entry.file_id, inv.revision_id), None
119
            except KeyError:
120
                ret = self._get_ie_object(entry, inv)
121
                self._idmap.add_entry(ret.id, "tree", (entry.file_id, inv.revision_id))
122
                return ret.id, ret
123
        else:
124
            try:
125
                return self._idmap.lookup_blob(entry.file_id, entry.revision), None
126
            except KeyError:
127
                ret = self._get_ie_object(entry, inv)
128
                self._idmap.add_entry(ret.id, "blob", (entry.file_id, entry.revision))
129
                return ret.id, ret
130
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
131
    def _get_ie_sha1(self, entry, inv):
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
132
        return self._get_ie_object_or_sha1(entry, inv)[0]
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
133
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
134
    def _get_blob(self, fileid, revision, expected_sha=None):
0.200.236 by Jelmer Vernooij
require bzr 1.13.
135
        """Return a Git Blob object from a fileid and revision stored in bzr.
136
        
137
        :param fileid: File id of the text
138
        :param revision: Revision of the text
139
        """
0.200.260 by Jelmer Vernooij
Add DictGitShaMap, useful for testing.
140
        text = self.repository.texts.get_record_stream([(fileid, revision)],
141
            "unordered", True).next().get_bytes_as("fulltext")
0.200.229 by Jelmer Vernooij
More work on converter.
142
        blob = Blob()
143
        blob._text = text
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
144
        self._check_expected_sha(expected_sha, blob)
0.200.229 by Jelmer Vernooij
More work on converter.
145
        return blob
146
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
147
    def _get_tree(self, fileid, revid, inv=None, expected_sha=None):
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
148
        """Return a Git Tree object from a file id and a revision stored in bzr.
0.200.249 by Jelmer Vernooij
Implement Tree.
149
0.200.343 by Jelmer Vernooij
Use file ids consistently in map.
150
        :param fileid: fileid in the tree.
0.200.249 by Jelmer Vernooij
Implement Tree.
151
        :param revision: Revision of the tree.
152
        """
153
        if inv is None:
154
            inv = self.repository.get_inventory(revid)
0.200.359 by Jelmer Vernooij
Simplify file mode handling, avoid inventory_to_tree_and_blobs as it is expensive if trees/blobs have already been converted.
155
        tree = directory_to_tree(inv[fileid], lambda ie: self._get_ie_sha1(ie, inv))
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
156
        self._check_expected_sha(expected_sha, tree)
0.200.249 by Jelmer Vernooij
Implement Tree.
157
        return tree
0.200.229 by Jelmer Vernooij
More work on converter.
158
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
159
    def _get_commit(self, revid, tree_sha, expected_sha=None):
0.200.229 by Jelmer Vernooij
More work on converter.
160
        rev = self.repository.get_revision(revid)
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
161
        commit = revision_to_commit(rev, tree_sha, self._lookup_revision_sha1)
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
162
        self._check_expected_sha(expected_sha, commit)
163
        return commit
0.200.228 by Jelmer Vernooij
Split out map.
164
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
165
    def get_parents(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
166
        """Retrieve the parents of a Git commit by SHA1.
167
168
        :param sha: SHA1 of the commit
169
        :raises: KeyError, NotCommitError
170
        """
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
171
        return self[sha].parents
172
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
173
    def _lookup_revision_sha1(self, revid):
0.200.449 by Jelmer Vernooij
Use BazaarObjectStore to find matching SHA1s for bzr revisions.
174
        """Return the SHA1 matching a Bazaar revision."""
0.200.364 by Jelmer Vernooij
Reimplement dpush, but more efficient and only writing a single pack file rather than one per revision.
175
        try:
176
            return self._idmap._parent_lookup(revid)
177
        except KeyError:
178
            inv = self.repository.get_inventory(revid)
179
            tree_sha = self._get_ie_sha1(inv.root, inv)
180
            ret = self._get_commit(revid, tree_sha).id
181
            self._idmap.add_entry(ret, "commit", (revid, tree_sha))
182
            return ret
183
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
184
    def get_raw(self, sha):
0.200.454 by Jelmer Vernooij
Use ObjectStore.find_missing_objects in server.
185
        """Get the raw representation of a Git object by SHA1.
186
187
        :param sha: SHA1 of the git object
188
        """
0.200.421 by Jelmer Vernooij
Use public method to access raw git object text.
189
        return self[sha].as_raw_string()
0.200.310 by Jelmer Vernooij
Fix pull from remote branches.
190
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
191
    def __contains__(self, sha):
192
        # See if sha is in map
193
        try:
194
            self._lookup_git_sha(sha)
195
        except KeyError:
196
            return False
197
        else:
198
            return True
199
200
    def _lookup_git_sha(self, sha):
201
        # See if sha is in map
202
        try:
203
            return self._idmap.lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
204
        except KeyError:
205
            # if not, see if there are any unconverted revisions and add them 
206
            # to the map, search for sha in map again
207
            self._update_sha_map()
0.200.437 by Jelmer Vernooij
Implement BazaarObjectStore.__contains__, BazaarObjectStore.iter_shas, BazaarObjectStore.get_parents.
208
            return self._idmap.lookup_git_sha(sha)
209
210
    def __getitem__(self, sha):
211
        (type, type_data) = self._lookup_git_sha(sha)
0.200.228 by Jelmer Vernooij
Split out map.
212
        # convert object to git object
0.200.229 by Jelmer Vernooij
More work on converter.
213
        if type == "commit":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
214
            return self._get_commit(type_data[0], type_data[1], 
215
                                    expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
216
        elif type == "blob":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
217
            return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
0.200.229 by Jelmer Vernooij
More work on converter.
218
        elif type == "tree":
0.200.353 by Jelmer Vernooij
fileids/revids are plain strings, not unicode
219
            return self._get_tree(type_data[0], type_data[1], expected_sha=sha)
0.200.228 by Jelmer Vernooij
Split out map.
220
        else:
221
            raise AssertionError("Unknown object type '%s'" % type)