/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to object_store.py

Proper branch names.

Show diffs side-by-side

added added

removed removed

Lines of Context:
object_store.py
1
 
# Copyright (C) 2009 Canonical Ltd
 
1
# Copyright (C) 2009 Jelmer Vernooij <jelmer@samba.org>
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
16
16
 
17
17
"""Map from Git sha's to Bazaar objects."""
18
18
 
19
 
import bzrlib
20
 
 
21
 
from bzrlib import ui
22
 
 
23
 
from bzrlib.errors import NoSuchRevision
24
 
 
25
 
from bzrlib.plugins.git.mapping import (
26
 
    inventory_to_tree_and_blobs,
27
 
    revision_to_commit,
28
 
    )
29
 
from bzrlib.plugins.git.shamap import GitShaMap
30
 
 
31
19
from dulwich.objects import (
32
20
    Blob,
33
 
    )
34
 
 
35
 
 
36
 
class GitObjectConverter(object):
 
21
    sha_to_hex,
 
22
    )
 
23
from dulwich.object_store import (
 
24
    BaseObjectStore,
 
25
    )
 
26
 
 
27
from bzrlib import (
 
28
    debug,
 
29
    errors,
 
30
    trace,
 
31
    ui,
 
32
    )
 
33
from bzrlib.revision import (
 
34
    NULL_REVISION,
 
35
    )
 
36
 
 
37
from bzrlib.plugins.git.mapping import (
 
38
    default_mapping,
 
39
    directory_to_tree,
 
40
    extract_unusual_modes,
 
41
    mapping_registry,
 
42
    )
 
43
from bzrlib.plugins.git.shamap import (
 
44
    from_repository as idmap_from_repository,
 
45
    )
 
46
 
 
47
 
 
48
def get_object_store(repo, mapping=None):
 
49
    git = getattr(repo, "_git", None)
 
50
    if git is not None:
 
51
        return git.object_store
 
52
    return BazaarObjectStore(repo, mapping)
 
53
 
 
54
 
 
55
class BazaarObjectStore(BaseObjectStore):
 
56
    """A Git-style object store backed onto a Bazaar repository."""
37
57
 
38
58
    def __init__(self, repository, mapping=None):
39
59
        self.repository = repository
40
60
        if mapping is None:
41
 
            self.mapping = self.repository.get_mapping()
 
61
            self.mapping = default_mapping
42
62
        else:
43
63
            self.mapping = mapping
44
 
        self._idmap = GitShaMap(self.repository._transport)
 
64
        self._idmap = idmap_from_repository(repository)
 
65
        self.start_write_group = self._idmap.start_write_group
 
66
        self.abort_write_group = self._idmap.abort_write_group
 
67
        self.commit_write_group = self._idmap.commit_write_group
45
68
 
46
 
    def _update_sha_map(self):
47
 
        all_revids = self.repository.all_revision_ids()
 
69
    def _update_sha_map(self, stop_revision=None):
48
70
        graph = self.repository.get_graph()
49
 
        present_revids = set(self._idmap.revids())
50
 
        pb = ui.ui_factory.nested_progress_bar()
 
71
        if stop_revision is None:
 
72
            heads = graph.heads(self.repository.all_revision_ids())
 
73
        else:
 
74
            heads = set([stop_revision])
 
75
        missing_revids = self._idmap.missing_revisions(heads)
 
76
        while heads:
 
77
            parents = graph.get_parent_map(heads)
 
78
            todo = set()
 
79
            for p in parents.values():
 
80
                todo.update([x for x in p if x not in missing_revids])
 
81
            heads = self._idmap.missing_revisions(todo)
 
82
            missing_revids.update(heads)
 
83
        if NULL_REVISION in missing_revids:
 
84
            missing_revids.remove(NULL_REVISION)
 
85
        missing_revids = self.repository.has_revisions(missing_revids)
 
86
        if not missing_revids:
 
87
            return
 
88
        self.start_write_group()
51
89
        try:
52
 
            for i, revid in enumerate(graph.iter_topo_order(all_revids)):
53
 
                if revid in present_revids:
54
 
                    continue
55
 
                pb.update("updating git map", i, len(all_revids))
56
 
                self._update_sha_map_revision(revid)
57
 
        finally:
58
 
            pb.finished()
 
90
            pb = ui.ui_factory.nested_progress_bar()
 
91
            try:
 
92
                for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
 
93
                    trace.mutter('processing %r', revid)
 
94
                    pb.update("updating git map", i, len(missing_revids))
 
95
                    self._update_sha_map_revision(revid)
 
96
            finally:
 
97
                pb.finished()
 
98
        except:
 
99
            self.abort_write_group()
 
100
            raise
 
101
        else:
 
102
            self.commit_write_group()
 
103
 
 
104
    def __iter__(self):
 
105
        self._update_sha_map()
 
106
        return iter(self._idmap.sha1s())
 
107
 
 
108
    def _revision_to_commit(self, rev, tree_sha):
 
109
        def parent_lookup(revid):
 
110
            try:
 
111
                return self._lookup_revision_sha1(revid)
 
112
            except errors.NoSuchRevision:
 
113
                trace.warning("Ignoring ghost parent %s", revid)
 
114
                return None
 
115
        return self.mapping.export_commit(rev, tree_sha, parent_lookup)
 
116
 
 
117
    def _inventory_to_objects(self, inv, invshamap, unusual_modes):
 
118
        tree_sha = self._get_ie_sha1(inv.root, inv, invshamap, unusual_modes)
 
119
        return tree_sha, []
59
120
 
60
121
    def _update_sha_map_revision(self, revid):
61
122
        inv = self.repository.get_inventory(revid)
62
 
        objects = inventory_to_tree_and_blobs(self.repository, self.mapping, revid)
63
 
        for sha, o, path in objects:
64
 
            if path == "":
65
 
                tree_sha = sha
66
 
            ie = inv[inv.path2id(path)]
67
 
            if ie.kind in ("file", "symlink"):
68
 
                self._idmap.add_entry(sha, "blob", (ie.file_id, ie.revision))
 
123
        rev = self.repository.get_revision(revid)
 
124
        unusual_modes = extract_unusual_modes(rev)
 
125
        invshamap = self._idmap.get_inventory_sha_map(revid)
 
126
        tree_sha, entries = self._inventory_to_objects(inv, invshamap,
 
127
            unusual_modes)
 
128
        commit_obj = self._revision_to_commit(rev, tree_sha)
 
129
        try:
 
130
            foreign_revid, mapping = mapping_registry.parse_revision_id(revid)
 
131
        except errors.InvalidRevisionId:
 
132
            pass
 
133
        else:
 
134
            if foreign_revid != commit_obj.id:
 
135
                if not "fix-shamap" in debug.debug_flags:
 
136
                    raise AssertionError("recreated git commit had different sha1: expected %s, got %s" % (foreign_revid, commit_obj.id))
 
137
        self._idmap.add_entries(revid, rev.parent_ids, commit_obj.id, 
 
138
            tree_sha, entries)
 
139
 
 
140
    def _check_expected_sha(self, expected_sha, object):
 
141
        if expected_sha is None:
 
142
            return
 
143
        if len(expected_sha) == 40:
 
144
            if expected_sha != object.sha().hexdigest():
 
145
                raise AssertionError("Invalid sha for %r: %s" % (object, expected_sha))
 
146
        elif len(expected_sha) == 20:
 
147
            if expected_sha != object.sha().digest():
 
148
                raise AssertionError("Invalid sha for %r: %s" % (object, sha_to_hex(expected_sha)))
 
149
        else:
 
150
            raise AssertionError("Unknown length %d for %r" % (len(expected_sha), expected_sha))
 
151
 
 
152
    def _get_ie_object(self, entry, inv, unusual_modes):
 
153
        if entry.kind == "directory":
 
154
            return self._get_tree(entry.file_id, inv.revision_id, inv,
 
155
                unusual_modes)
 
156
        elif entry.kind == "symlink":
 
157
            return self._get_blob_for_symlink(entry.symlink_target)
 
158
        elif entry.kind == "file":
 
159
            return self._get_blob_for_file(entry.file_id, entry.revision)
 
160
        else:
 
161
            raise AssertionError("unknown entry kind '%s'" % entry.kind)
 
162
 
 
163
    def _get_ie_object_or_sha1(self, entry, inv, invshamap, unusual_modes):
 
164
        if entry.kind == "directory":
 
165
            try:
 
166
                return invshamap.lookup_tree(entry.file_id), None
 
167
            except (KeyError, NotImplementedError):
 
168
                ret = self._get_ie_object(entry, inv, unusual_modes)
 
169
                if ret is None:
 
170
                    # Empty directory
 
171
                    hexsha = None
 
172
                else:
 
173
                    hexsha = ret.id
 
174
                self._idmap._add_entry(hexsha, "tree",
 
175
                    (entry.file_id, inv.revision_id))
 
176
                return hexsha, ret
 
177
        elif entry.kind in ("file", "symlink"):
 
178
            try:
 
179
                return invshamap.lookup_blob(entry.file_id, entry.revision), None
 
180
            except KeyError:
 
181
                ret = self._get_ie_object(entry, inv, unusual_modes)
 
182
                self._idmap._add_entry(ret.id, "blob", (entry.file_id,
 
183
                    entry.revision))
 
184
                return ret.id, ret
 
185
        else:
 
186
            raise AssertionError("unknown entry kind '%s'" % entry.kind)
 
187
 
 
188
    def _get_ie_sha1(self, entry, inv, invshamap, unusual_modes):
 
189
        return self._get_ie_object_or_sha1(entry, inv, invshamap, unusual_modes)[0]
 
190
 
 
191
    def _get_blob_for_symlink(self, symlink_target, expected_sha=None):
 
192
        """Return a Git Blob object for symlink.
 
193
 
 
194
        :param symlink_target: target of symlink.
 
195
        """
 
196
        if type(symlink_target) == unicode:
 
197
            symlink_target = symlink_target.encode('utf-8')
 
198
        blob = Blob()
 
199
        blob._text = symlink_target
 
200
        self._check_expected_sha(expected_sha, blob)
 
201
        return blob
 
202
 
 
203
    def _get_blob_for_file(self, fileid, revision, expected_sha=None):
 
204
        """Return a Git Blob object from a fileid and revision stored in bzr.
 
205
 
 
206
        :param fileid: File id of the text
 
207
        :param revision: Revision of the text
 
208
        """
 
209
        blob = Blob()
 
210
        chunks = self.repository.iter_files_bytes([(fileid, revision, None)]).next()[1]
 
211
        blob._text = "".join(chunks)
 
212
        self._check_expected_sha(expected_sha, blob)
 
213
        return blob
 
214
 
 
215
    def _get_blob(self, fileid, revision, expected_sha=None):
 
216
        """Return a Git Blob object from a fileid and revision stored in bzr.
 
217
 
 
218
        :param fileid: File id of the text
 
219
        :param revision: Revision of the text
 
220
        """
 
221
        inv = self.repository.get_inventory(revision)
 
222
        entry = inv[fileid]
 
223
 
 
224
        if entry.kind == 'file':
 
225
            return self._get_blob_for_file(entry.file_id, entry.revision,
 
226
                                           expected_sha=expected_sha)
 
227
        elif entry.kind == 'symlink':
 
228
            return self._get_blob_for_symlink(entry.symlink_target,
 
229
                                              expected_sha=expected_sha)
 
230
        else:
 
231
            raise AssertionError
 
232
 
 
233
    def _get_tree(self, fileid, revid, inv, unusual_modes, expected_sha=None):
 
234
        """Return a Git Tree object from a file id and a revision stored in bzr.
 
235
 
 
236
        :param fileid: fileid in the tree.
 
237
        :param revision: Revision of the tree.
 
238
        """
 
239
        invshamap = self._idmap.get_inventory_sha_map(inv.revision_id)
 
240
        tree = directory_to_tree(inv[fileid],
 
241
            lambda ie: self._get_ie_sha1(ie, inv, invshamap, unusual_modes),
 
242
            unusual_modes)
 
243
        self._check_expected_sha(expected_sha, tree)
 
244
        return tree
 
245
 
 
246
    def _get_commit(self, rev, tree_sha, expected_sha=None):
 
247
        commit = self._revision_to_commit(rev, tree_sha)
 
248
        self._check_expected_sha(expected_sha, commit)
 
249
        return commit
 
250
 
 
251
    def get_parents(self, sha):
 
252
        """Retrieve the parents of a Git commit by SHA1.
 
253
 
 
254
        :param sha: SHA1 of the commit
 
255
        :raises: KeyError, NotCommitError
 
256
        """
 
257
        return self[sha].parents
 
258
 
 
259
    def _lookup_revision_sha1(self, revid):
 
260
        """Return the SHA1 matching a Bazaar revision."""
 
261
        if revid == NULL_REVISION:
 
262
            return "0" * 40
 
263
        try:
 
264
            return self._idmap.lookup_commit(revid)
 
265
        except KeyError:
 
266
            try:
 
267
                return mapping_registry.parse_revision_id(revid)[0]
 
268
            except errors.InvalidRevisionId:
 
269
                self._update_sha_map(revid)
 
270
                return self._idmap.lookup_commit(revid)
 
271
 
 
272
    def get_raw(self, sha):
 
273
        """Get the raw representation of a Git object by SHA1.
 
274
 
 
275
        :param sha: SHA1 of the git object
 
276
        """
 
277
        obj = self[sha]
 
278
        return (obj.type, obj.as_raw_string())
 
279
 
 
280
    def __contains__(self, sha):
 
281
        # See if sha is in map
 
282
        try:
 
283
            (type, type_data) = self._lookup_git_sha(sha)
 
284
            if type == "commit":
 
285
                return self.repository.has_revision(type_data[0])
 
286
            elif type == "blob":
 
287
                return self.repository.texts.has_version(type_data)
 
288
            elif type == "tree":
 
289
                return self.repository.has_revision(type_data[1])
69
290
            else:
70
 
                self._idmap.add_entry(sha, "tree", (ie.file_id, ie.revision))
71
 
        rev = self.repository.get_revision(revid)
72
 
        commit_obj = revision_to_commit(rev, tree_sha, self._idmap._parent_lookup)
73
 
        self._idmap.add_entry(commit_obj.sha().hexdigest(), "commit", (revid, tree_sha))
74
 
 
75
 
    def _get_blob(self, fileid, revision):
76
 
        text = self.repository.texts.get_record_stream([(fileid, revision)], "unordered", True).next().get_bytes_as("fulltext")
77
 
        blob = Blob()
78
 
        blob._text = text
79
 
        return blob
80
 
 
81
 
    def _get_tree(self, fileid, revid):
82
 
        raise NotImplementedError(self._get_tree)
83
 
 
84
 
    def _get_commit(self, revid, tree_sha):
85
 
        rev = self.repository.get_revision(revid)
86
 
        return revision_to_commit(rev, tree_sha, self._idmap._parent_lookup)
87
 
 
88
 
    def __getitem__(self, sha):
 
291
                raise AssertionError("Unknown object type '%s'" % type)
 
292
        except KeyError:
 
293
            return False
 
294
 
 
295
    def _lookup_git_sha(self, sha):
89
296
        # See if sha is in map
90
297
        try:
91
 
            (type, type_data) = self._idmap.lookup_git_sha(sha)
 
298
            return self._idmap.lookup_git_sha(sha)
92
299
        except KeyError:
93
 
            # if not, see if there are any unconverted revisions and add them 
 
300
            # if not, see if there are any unconverted revisions and add them
94
301
            # to the map, search for sha in map again
95
302
            self._update_sha_map()
96
 
            (type, type_data) = self._idmap.lookup_git_sha(sha)
 
303
            return self._idmap.lookup_git_sha(sha)
 
304
 
 
305
    def __getitem__(self, sha):
 
306
        (type, type_data) = self._lookup_git_sha(sha)
97
307
        # convert object to git object
98
308
        if type == "commit":
99
 
            return self._get_commit(*type_data)
 
309
            try:
 
310
                rev = self.repository.get_revision(type_data[0])
 
311
            except errors.NoSuchRevision:
 
312
                trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
 
313
                raise KeyError(sha)
 
314
            return self._get_commit(rev, type_data[1], expected_sha=sha)
100
315
        elif type == "blob":
101
 
            return self._get_blob(*type_data)
 
316
            return self._get_blob(type_data[0], type_data[1], expected_sha=sha)
102
317
        elif type == "tree":
103
 
            return self._get_tree(*type_data)
 
318
            try:
 
319
                inv = self.repository.get_inventory(type_data[1])
 
320
                rev = self.repository.get_revision(type_data[1])
 
321
            except errors.NoSuchRevision:
 
322
                trace.mutter('entry for %s %s in shamap: %r, but not found in repository', type, sha, type_data)
 
323
                raise KeyError(sha)
 
324
            unusual_modes = extract_unusual_modes(rev)
 
325
            try:
 
326
                return self._get_tree(type_data[0], type_data[1], inv,
 
327
                    unusual_modes, expected_sha=sha)
 
328
            except errors.NoSuchRevision:
 
329
                raise KeyError(sha)
104
330
        else:
105
331
            raise AssertionError("Unknown object type '%s'" % type)