/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to repository.py

Add more tests for fetch code.

Show diffs side-by-side

added added

removed removed

Lines of Context:
repository.py
1
1
# Copyright (C) 2007 Canonical Ltd
 
2
# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
2
3
#
3
4
# This program is free software; you can redistribute it and/or modify
4
5
# it under the terms of the GNU General Public License as published by
16
17
 
17
18
"""An adapter between a Git Repository and a Bazaar Branch"""
18
19
 
19
 
import git
 
20
import dulwich as git
20
21
import os
21
22
import time
22
23
 
23
24
import bzrlib
24
25
from bzrlib import (
25
 
    deprecated_graph,
26
26
    errors,
 
27
    graph,
27
28
    inventory,
28
29
    osutils,
29
30
    repository,
30
31
    revision,
31
32
    revisiontree,
 
33
    ui,
32
34
    urlutils,
33
 
    versionedfile,
34
35
    )
 
36
from bzrlib.foreign import (
 
37
        ForeignRepository,
 
38
        )
 
39
from bzrlib.trace import mutter
35
40
from bzrlib.transport import get_transport
36
41
 
37
 
from bzrlib.plugins.git import (
38
 
    cache,
39
 
    ids,
40
 
    versionedfiles
41
 
    )
42
 
 
43
 
 
44
 
class GitRepository(repository.Repository):
 
42
from bzrlib.plugins.git.foreign import (
 
43
    versionedfiles,
 
44
    )
 
45
from bzrlib.plugins.git.mapping import (
 
46
    default_mapping,
 
47
    inventory_to_tree_and_blobs,
 
48
    mapping_registry,
 
49
    revision_to_commit,
 
50
    )
 
51
from bzrlib.plugins.git.versionedfiles import GitTexts
 
52
 
 
53
 
 
54
class GitTags(object):
 
55
 
 
56
    def __init__(self, tags):
 
57
        self._tags = tags
 
58
 
 
59
    def __iter__(self):
 
60
        return iter(self._tags)
 
61
 
 
62
 
 
63
class GitRepository(ForeignRepository):
45
64
    """An adapter to git repositories for bzr."""
46
65
 
47
66
    _serializer = None
48
67
 
49
68
    def __init__(self, gitdir, lockfiles):
 
69
        ForeignRepository.__init__(self, GitFormat(), gitdir, lockfiles)
 
70
        from bzrlib.plugins.git import fetch
 
71
        repository.InterRepository.register_optimiser(fetch.InterGitRepository)
 
72
        repository.InterRepository.register_optimiser(fetch.InterGitNonGitRepository)
 
73
 
 
74
    def is_shared(self):
 
75
        return True
 
76
 
 
77
    def supports_rich_root(self):
 
78
        return True
 
79
 
 
80
    def _warn_if_deprecated(self):
 
81
        # This class isn't deprecated
 
82
        pass
 
83
 
 
84
    def get_mapping(self):
 
85
        return default_mapping
 
86
 
 
87
    def make_working_trees(self):
 
88
        return True
 
89
 
 
90
 
 
91
class LocalGitRepository(GitRepository):
 
92
 
 
93
    def __init__(self, gitdir, lockfiles):
 
94
        # FIXME: This also caches negatives. Need to be more careful 
 
95
        # about this once we start writing to git
 
96
        self._parents_provider = graph.CachingParentsProvider(self)
 
97
        GitRepository.__init__(self, gitdir, lockfiles)
50
98
        self.base = gitdir.root_transport.base
51
 
        self.bzrdir = gitdir
52
 
        self.control_files = lockfiles
53
99
        self._git = gitdir._git
54
 
        cache_dir = cache.create_cache_dir()
55
 
        cachedir_transport = get_transport(cache_dir)
56
 
        cache_file = os.path.join(cache_dir, 'cache-%s' % ids.NAMESPACE)
57
100
        self.texts = None
58
101
        self.signatures = versionedfiles.VirtualSignatureTexts(self)
59
102
        self.revisions = versionedfiles.VirtualRevisionTexts(self)
60
 
        self._format = GitFormat()
61
 
        self._fallback_repositories = []
 
103
        self.inventories = versionedfiles.VirtualInventoryTexts(self)
 
104
        self.texts = GitTexts(self)
 
105
        self.tags = GitTags(self._git.get_tags())
62
106
 
63
 
    def _all_revision_ids(self):
64
 
        if self._git.heads == []:
65
 
            return set()
66
 
        ret = set()
67
 
        skip = 0
68
 
        max_count = 1000
69
 
        cms = None
70
 
        while cms != []:
71
 
            cms = self._git.commits("--all", max_count=max_count, skip=skip)
72
 
            skip += max_count
73
 
            ret.update([ids.convert_revision_id_git_to_bzr(cm.id) for cm in cms])
 
107
    def all_revision_ids(self):
 
108
        ret = set([revision.NULL_REVISION])
 
109
        heads = self._git.heads()
 
110
        if heads == {}:
 
111
            return ret
 
112
        bzr_heads = [self.get_mapping().revision_id_foreign_to_bzr(h) for h in heads.itervalues()]
 
113
        ret = set(bzr_heads)
 
114
        graph = self.get_graph()
 
115
        for rev, parents in graph.iter_ancestry(bzr_heads):
 
116
            ret.add(rev)
74
117
        return ret
75
118
 
76
 
    def is_shared(self):
77
 
        return True
78
 
 
79
 
    def supports_rich_root(self):
80
 
        return False
81
 
 
82
119
    #def get_revision_delta(self, revision_id):
83
120
    #    parent_revid = self.get_revision(revision_id).parent_ids[0]
84
121
    #    diff = self._git.diff(ids.convert_revision_id_bzr_to_git(parent_revid),
85
122
    #                   ids.convert_revision_id_bzr_to_git(revision_id))
86
123
 
87
 
    def get_ancestry(self, revision_id):
88
 
        revision_id = revision.ensure_null(revision_id)
89
 
        ret = []
90
 
        if revision_id != revision.NULL_REVISION:
91
 
            skip = 0
92
 
            max_count = 1000
93
 
            cms = None
94
 
            while cms != []:
95
 
                cms = self._git.commits(ids.convert_revision_id_bzr_to_git(revision_id), max_count=max_count, skip=skip)
96
 
                skip += max_count
97
 
                ret += [ids.convert_revision_id_git_to_bzr(cm.id) for cm in cms]
98
 
        return [None] + ret
 
124
    def _make_parents_provider(self):
 
125
        """See Repository._make_parents_provider()."""
 
126
        return self._parents_provider
 
127
 
 
128
    def get_parent_map(self, revids):
 
129
        parent_map = {}
 
130
        mutter("get_parent_map(%r)", revids)
 
131
        for revision_id in revids:
 
132
            assert isinstance(revision_id, str)
 
133
            if revision_id == revision.NULL_REVISION:
 
134
                parent_map[revision_id] = ()
 
135
                continue
 
136
            hexsha, mapping = self.lookup_git_revid(revision_id)
 
137
            commit  = self._git.commit(hexsha)
 
138
            if commit is None:
 
139
                continue
 
140
            else:
 
141
                parent_map[revision_id] = [mapping.revision_id_foreign_to_bzr(p) for p in commit.parents]
 
142
        return parent_map
 
143
 
 
144
    def get_ancestry(self, revision_id, topo_sorted=True):
 
145
        """See Repository.get_ancestry().
 
146
        """
 
147
        if revision_id is None:
 
148
            return [None, revision.NULL_REVISION] + self._all_revision_ids()
 
149
        assert isinstance(revision_id, str)
 
150
        ancestry = []
 
151
        graph = self.get_graph()
 
152
        for rev, parents in graph.iter_ancestry([revision_id]):
 
153
            ancestry.append(rev)
 
154
        ancestry.reverse()
 
155
        return [None] + ancestry
 
156
 
 
157
    def import_revision_gist(self, source, revid, parent_lookup):
 
158
        """Import the gist of a revision into this Git repository.
 
159
 
 
160
        """
 
161
        objects = []
 
162
        rev = source.get_revision(revid)
 
163
        for sha, object, path in inventory_to_tree_and_blobs(source, None, revid):
 
164
            if path == "":
 
165
                tree_sha = sha
 
166
            objects.append((object, path))
 
167
        commit = revision_to_commit(rev, tree_sha, parent_lookup)
 
168
        objects.append((commit, None))
 
169
        self._git.object_store.add_objects(objects)
 
170
        return commit.sha().hexdigest()
 
171
 
 
172
    def dfetch(self, source, stop_revision):
 
173
        """Import the gist of the ancestry of a particular revision."""
 
174
        if stop_revision is None:
 
175
            raise NotImplementedError
 
176
        revidmap = {}
 
177
        gitidmap = {}
 
178
        todo = []
 
179
        source.lock_write()
 
180
        try:
 
181
            graph = source.get_graph()
 
182
            ancestry = [x for x in source.get_ancestry(stop_revision) if x is not None]
 
183
            for revid in graph.iter_topo_order(ancestry):
 
184
                if not self.has_revision(revid):
 
185
                    todo.append(revid)
 
186
            pb = ui.ui_factory.nested_progress_bar()
 
187
            try:
 
188
                for i, revid in enumerate(todo):
 
189
                    pb.update("pushing revisions", i, len(todo))
 
190
                    git_commit = self.import_revision_gist(source, revid, gitidmap.__getitem__)
 
191
                    gitidmap[revid] = git_commit
 
192
                    git_revid = self.get_mapping().revision_id_foreign_to_bzr(git_commit)
 
193
                    revidmap[revid] = git_revid
 
194
            finally:
 
195
                pb.finished()
 
196
            source.fetch(self, revision_id=revidmap[stop_revision])
 
197
        finally:
 
198
            source.unlock()
 
199
        return revidmap
99
200
 
100
201
    def get_signature_text(self, revision_id):
101
202
        raise errors.NoSuchRevision(self, revision_id)
102
203
 
 
204
    def lookup_revision_id(self, revid):
 
205
        """Lookup a revision id.
 
206
        
 
207
        :param revid: Bazaar revision id.
 
208
        :return: Tuple with git revisionid and mapping.
 
209
        """
 
210
        # Yes, this doesn't really work, but good enough as a stub
 
211
        return osutils.sha(rev_id).hexdigest(), self.get_mapping()
 
212
 
103
213
    def has_signature_for_revision_id(self, revision_id):
104
214
        return False
105
215
 
106
 
    def get_parent_map(self, revision_ids):
107
 
        ret = {}
108
 
        for revid in revision_ids:
109
 
            if revid == revision.NULL_REVISION:
110
 
                ret[revid] = ()
111
 
            else:
112
 
                commit = self._git.commit(ids.convert_revision_id_bzr_to_git(revid))
113
 
                ret[revid] = tuple([ids.convert_revision_id_git_to_bzr(p.id) for p in commit.parents])
114
 
        return ret
 
216
    def lookup_git_revid(self, bzr_revid):
 
217
        try:
 
218
            return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
 
219
        except errors.InvalidRevisionId:
 
220
            raise errors.NoSuchRevision(self, bzr_revid)
115
221
 
116
222
    def get_revision(self, revision_id):
117
 
        git_commit_id = ids.convert_revision_id_bzr_to_git(revision_id)
118
 
        commit = self._git.commit(git_commit_id)
 
223
        git_commit_id, mapping = self.lookup_git_revid(revision_id)
 
224
        try:
 
225
            commit = self._git.commit(git_commit_id)
 
226
        except KeyError:
 
227
            raise errors.NoSuchRevision(self, revision_id)
119
228
        # print "fetched revision:", git_commit_id
120
 
        revision = self._parse_rev(commit)
 
229
        revision = mapping.import_commit(commit)
 
230
        assert revision is not None
121
231
        return revision
122
232
 
123
233
    def has_revision(self, revision_id):
124
234
        try:
125
235
            self.get_revision(revision_id)
126
 
        except NoSuchRevision:
 
236
        except errors.NoSuchRevision:
127
237
            return False
128
238
        else:
129
239
            return True
130
240
 
131
 
    def get_revisions(self, revisions):
132
 
        return [self.get_revision(r) for r in revisions]
133
 
 
134
 
    @classmethod
135
 
    def _parse_rev(klass, commit):
136
 
        """Convert a git commit to a bzr revision.
137
 
 
138
 
        :return: a `bzrlib.revision.Revision` object.
139
 
        """
140
 
        rev = revision.Revision(ids.convert_revision_id_git_to_bzr(commit.id))
141
 
        rev.parent_ids = tuple([ids.convert_revision_id_git_to_bzr(p.id) for p in commit.parents])
142
 
        rev.inventory_sha1 = ""
143
 
        rev.message = commit.message.decode("utf-8", "replace")
144
 
        rev.committer = str(commit.committer)
145
 
        rev.properties['author'] = str(commit.author)
146
 
        rev.timestamp = time.mktime(commit.committed_date)
147
 
        rev.timezone = 0
148
 
        return rev
 
241
    def get_revisions(self, revids):
 
242
        return [self.get_revision(r) for r in revids]
149
243
 
150
244
    def revision_trees(self, revids):
151
245
        for revid in revids:
165
259
        assert revision_id != None
166
260
        return self.revision_tree(revision_id).inventory
167
261
 
168
 
 
169
 
def escape_file_id(file_id):
170
 
    return file_id.replace('_', '__').replace(' ', '_s')
171
 
 
172
 
 
173
 
def unescape_file_id(file_id):
174
 
    return file_id.replace("_s", " ").replace("__", "_")
 
262
    def set_make_working_trees(self, trees):
 
263
        pass
 
264
 
 
265
    def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref, progress=None):
 
266
        return self._git.fetch_objects(determine_wants, graph_walker, progress)
175
267
 
176
268
 
177
269
class GitRevisionTree(revisiontree.RevisionTree):
179
271
    def __init__(self, repository, revision_id):
180
272
        self._repository = repository
181
273
        self.revision_id = revision_id
182
 
        git_id = ids.convert_revision_id_bzr_to_git(revision_id)
183
 
        self.tree = repository._git.commit(git_id).tree
 
274
        assert isinstance(revision_id, str)
 
275
        git_id, self.mapping = repository.lookup_git_revid(revision_id)
 
276
        try:
 
277
            commit = repository._git.commit(git_id)
 
278
        except KeyError, r:
 
279
            raise errors.NoSuchRevision(repository, revision_id)
 
280
        self.tree = commit.tree
184
281
        self._inventory = inventory.Inventory(revision_id=revision_id)
185
282
        self._inventory.root.revision = revision_id
186
283
        self._build_inventory(self.tree, self._inventory.root, "")
191
288
    def get_file_text(self, file_id):
192
289
        entry = self._inventory[file_id]
193
290
        if entry.kind == 'directory': return ""
194
 
        return self._repository._git.blob(entry.text_id).data
 
291
        return self._repository._git.get_blob(entry.text_id).data
195
292
 
196
 
    def _build_inventory(self, tree, ie, path):
 
293
    def _build_inventory(self, tree_id, ie, path):
197
294
        assert isinstance(path, str)
198
 
        for b in tree.contents:
199
 
            basename = b.name.decode("utf-8")
 
295
        tree = self._repository._git.tree(tree_id)
 
296
        for mode, name, hexsha in tree.entries():
 
297
            basename = name.decode("utf-8")
200
298
            if path == "":
201
 
                child_path = b.name
 
299
                child_path = name
202
300
            else:
203
 
                child_path = urlutils.join(path, b.name)
204
 
            file_id = escape_file_id(child_path.encode('utf-8'))
205
 
            if b.mode[0] == '0':
 
301
                child_path = urlutils.join(path, name)
 
302
            file_id = self.mapping.generate_file_id(child_path)
 
303
            entry_kind = (mode & 0700000) / 0100000
 
304
            if entry_kind == 0:
206
305
                child_ie = inventory.InventoryDirectory(file_id, basename, ie.file_id)
207
 
            elif b.mode[0] == '1':
208
 
                if b.mode[1] == '0':
 
306
            elif entry_kind == 1:
 
307
                file_kind = (mode & 070000) / 010000
 
308
                b = self._repository._git.get_blob(hexsha)
 
309
                if file_kind == 0:
209
310
                    child_ie = inventory.InventoryFile(file_id, basename, ie.file_id)
210
311
                    child_ie.text_sha1 = osutils.sha_string(b.data)
211
 
                elif b.mode[1] == '2':
 
312
                elif file_kind == 2:
212
313
                    child_ie = inventory.InventoryLink(file_id, basename, ie.file_id)
213
314
                    child_ie.text_sha1 = osutils.sha_string("")
214
315
                else:
215
316
                    raise AssertionError(
216
 
                        "Unknown file kind, perms=%r." % (b.mode,))
 
317
                        "Unknown file kind, perms=%o." % (mode,))
217
318
                child_ie.text_id = b.id
218
 
                child_ie.text_size = b.size
 
319
                child_ie.text_size = len(b.data)
219
320
            else:
220
321
                raise AssertionError(
221
 
                    "Unknown blob kind, perms=%r." % (b.mode,))
222
 
            child_ie.executable = bool(int(b.mode[3:], 8) & 0111)
 
322
                    "Unknown blob kind, perms=%r." % (mode,))
 
323
            fs_mode = mode & 0777
 
324
            child_ie.executable = bool(fs_mode & 0111)
223
325
            child_ie.revision = self.revision_id
224
326
            self._inventory.add(child_ie)
225
 
            if b.mode[0] == '0':
226
 
                self._build_inventory(b, child_ie, child_path)
 
327
            if entry_kind == 0:
 
328
                self._build_inventory(hexsha, child_ie, child_path)
227
329
 
228
330
 
229
331
class GitFormat(object):
230
332
 
231
333
    supports_tree_reference = False
 
334
    rich_root_data = True
232
335
 
233
336
    def get_format_description(self):
234
337
        return "Git Repository"
 
338
 
 
339
    def initialize(self, url, shared=False, _internal=False):
 
340
        raise bzr_errors.UninitializableFormat(self)
 
341
 
 
342
    def check_conversion_target(self, target_repo_format):
 
343
        return target_repo_format.rich_root_data