/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to fetch.py

Fix branch tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
fetch.py
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
16
 
17
17
from bzrlib import osutils, ui, urlutils
18
 
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
 
18
from bzrlib.errors import InvalidRevisionId
19
19
from bzrlib.inventory import Inventory
20
20
from bzrlib.repository import InterRepository
21
21
from bzrlib.trace import info
22
22
from bzrlib.tsort import topo_sort
23
23
 
24
 
from bzrlib.plugins.git.repository import (
25
 
        LocalGitRepository, 
26
 
        GitRepository, 
27
 
        GitFormat,
28
 
        )
29
 
from bzrlib.plugins.git.converter import GitObjectConverter
 
24
from bzrlib.plugins.git import git
 
25
from bzrlib.plugins.git.repository import LocalGitRepository, GitRepository, GitFormat
30
26
from bzrlib.plugins.git.remote import RemoteGitRepository
31
27
 
32
 
import dulwich as git
33
 
from dulwich.client import SimpleFetchGraphWalker
34
28
from dulwich.objects import Commit
35
29
 
36
30
from cStringIO import StringIO
37
31
 
38
32
 
39
33
class BzrFetchGraphWalker(object):
40
 
    """GraphWalker implementation that uses a Bazaar repository."""
41
34
 
42
35
    def __init__(self, repository, mapping):
43
36
        self.repository = repository
46
39
        self.heads = set(repository.all_revision_ids())
47
40
        self.parents = {}
48
41
 
49
 
    def __iter__(self):
50
 
        return iter(self.next, None)
51
 
 
52
42
    def ack(self, sha):
53
43
        revid = self.mapping.revision_id_foreign_to_bzr(sha)
54
44
        self.remove(revid)
55
45
 
56
46
    def remove(self, revid):
57
47
        self.done.add(revid)
58
 
        if revid in self.heads:
 
48
        if ref in self.heads:
59
49
            self.heads.remove(revid)
60
50
        if revid in self.parents:
61
51
            for p in self.parents[revid]:
69
59
            self.heads.update([p for p in ps if not p in self.done])
70
60
            try:
71
61
                self.done.add(ret)
72
 
                return self.mapping.revision_id_bzr_to_foreign(ret)[0]
 
62
                return self.mapping.revision_id_bzr_to_foreign(ret)
73
63
            except InvalidRevisionId:
74
64
                pass
75
65
        return None
76
66
 
77
67
 
78
 
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, gitmap, executable):
 
68
def import_git_blob(repo, mapping, path, blob, inv, parent_invs, executable):
79
69
    """Import a git blob object into a bzr repository.
80
70
 
81
71
    :param repo: bzr repository
92
82
    ie.text_size = len(blob.data)
93
83
    ie.text_sha1 = osutils.sha_string(blob.data)
94
84
    ie.executable = executable
95
 
    gitmap._idmap.add_entry(blob.sha().hexdigest(), "blob", (ie.file_id, ie.revision))
96
 
 
97
 
 
98
 
def import_git_tree(repo, mapping, path, tree, inv, parent_invs, 
99
 
                    gitmap, lookup_object):
 
85
 
 
86
 
 
87
def import_git_tree(repo, mapping, path, tree, inv, parent_invs, lookup_object):
100
88
    """Import a git tree object into a bzr repository.
101
89
 
102
90
    :param repo: A Bzr repository object
111
99
        [])
112
100
    ie = inv.add_path(path, "directory", file_id)
113
101
    ie.revision = text_revision
114
 
    gitmap._idmap.add_entry(tree.sha().hexdigest(), "tree", (file_id, text_revision))
115
102
    for mode, name, hexsha in tree.entries():
116
103
        entry_kind = (mode & 0700000) / 0100000
117
104
        basename = name.decode("utf-8")
121
108
            child_path = urlutils.join(path, name)
122
109
        if entry_kind == 0:
123
110
            tree = lookup_object(hexsha)
124
 
            import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, gitmap, lookup_object)
 
111
            import_git_tree(repo, mapping, child_path, tree, inv, parent_invs, lookup_object)
125
112
        elif entry_kind == 1:
126
113
            blob = lookup_object(hexsha)
127
114
            fs_mode = mode & 0777
128
 
            import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, gitmap, bool(fs_mode & 0111))
 
115
            import_git_blob(repo, mapping, child_path, blob, inv, parent_invs, bool(fs_mode & 0111))
129
116
        else:
130
117
            raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
131
118
 
132
119
 
133
 
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever, 
134
 
        pb=None):
 
120
def import_git_objects(repo, mapping, object_iter, pb=None):
135
121
    """Import a set of git objects into a bzr repository.
136
122
 
137
123
    :param repo: Bazaar repository
139
125
    :param object_iter: Iterator over Git objects.
140
126
    """
141
127
    # TODO: a more (memory-)efficient implementation of this
 
128
    objects = {}
 
129
    for i, o in enumerate(object_iter):
 
130
        if pb is not None:
 
131
            pb.update("fetching objects", i) 
 
132
        objects[o.id] = o
142
133
    graph = []
143
134
    root_trees = {}
144
135
    revisions = {}
145
136
    # Find and convert commit objects
146
 
    for o in object_iter.iterobjects():
 
137
    for o in objects.itervalues():
147
138
        if isinstance(o, Commit):
148
139
            rev = mapping.import_commit(o)
149
 
            root_trees[rev.revision_id] = object_iter[o.tree]
 
140
            root_trees[rev.revision_id] = objects[o.tree]
150
141
            revisions[rev.revision_id] = rev
151
142
            graph.append((rev.revision_id, rev.parent_ids))
152
 
            target_git_object_retriever._idmap.add_entry(o.sha().hexdigest(), "commit", (rev.revision_id, o._tree))
153
143
    # Order the revisions
154
144
    # Create the inventory objects
155
145
    for i, revid in enumerate(topo_sort(graph)):
163
153
        inv = Inventory()
164
154
        inv.revision_id = rev.revision_id
165
155
        def lookup_object(sha):
166
 
            if sha in object_iter:
167
 
                return object_iter[sha]
168
 
            return target_git_object_retriever[sha]
 
156
            if sha in objects:
 
157
                return objects[sha]
 
158
            return reconstruct_git_object(repo, mapping, sha)
169
159
        parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
170
 
        import_git_tree(repo, mapping, "", root_tree, inv, parent_invs, 
171
 
            target_git_object_retriever, lookup_object)
 
160
        import_git_tree(repo, mapping, "", root_tree, inv, parent_invs, lookup_object)
172
161
        repo.add_revision(rev.revision_id, rev, inv)
173
162
 
174
163
 
175
 
class InterGitNonGitRepository(InterRepository):
 
164
def reconstruct_git_commit(repo, rev):
 
165
    raise NotImplementedError(self.reconstruct_git_commit)
 
166
 
 
167
 
 
168
def reconstruct_git_object(repo, mapping, sha):
 
169
    # Commit
 
170
    revid = mapping.revision_id_foreign_to_bzr(sha)
 
171
    try:
 
172
        rev = repo.get_revision(revid)
 
173
    except NoSuchRevision:
 
174
        pass
 
175
    else:
 
176
        return reconstruct_git_commit(rev)
 
177
 
 
178
    # TODO: Tree
 
179
    # TODO: Blob
 
180
    raise KeyError("No such object %s" % sha)
 
181
 
 
182
 
 
183
class InterGitRepository(InterRepository):
176
184
 
177
185
    _matching_repo_format = GitFormat()
178
186
 
184
192
        """See InterRepository.copy_content."""
185
193
        self.fetch(revision_id, pb, find_ghosts=False)
186
194
 
187
 
    def fetch_objects(self, determine_wants, mapping, pb=None):
188
 
        def progress(text):
189
 
            pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
190
 
        graph_walker = BzrFetchGraphWalker(self.target, mapping)
191
 
        create_pb = None
192
 
        if pb is None:
193
 
            create_pb = pb = ui.ui_factory.nested_progress_bar()
194
 
        target_git_object_retriever = GitObjectConverter(self.target, mapping)
195
 
        
196
 
        try:
197
 
            self.target.lock_write()
198
 
            try:
199
 
                self.target.start_write_group()
200
 
                try:
201
 
                    objects_iter = self.source.fetch_objects(determine_wants, 
202
 
                                graph_walker, 
203
 
                                target_git_object_retriever.__getitem__, 
204
 
                                progress)
205
 
                    import_git_objects(self.target, mapping, objects_iter, 
206
 
                            target_git_object_retriever, pb)
207
 
                finally:
208
 
                    self.target.commit_write_group()
209
 
            finally:
210
 
                self.target.unlock()
211
 
        finally:
212
 
            if create_pb:
213
 
                create_pb.finished()
214
 
 
215
195
    def fetch(self, revision_id=None, pb=None, find_ghosts=False, 
216
196
              mapping=None):
217
197
        if mapping is None:
218
198
            mapping = self.source.get_mapping()
 
199
        def progress(text):
 
200
            if pb is not None:
 
201
                pb.note("git: %s" % text)
 
202
            else:
 
203
                info("git: %s" % text)
219
204
        def determine_wants(heads):
220
205
            if revision_id is None:
221
206
                ret = heads.values()
222
207
            else:
223
 
                ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
 
208
                ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
224
209
            return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
225
 
        return self.fetch_objects(determine_wants, mapping, pb)
 
210
        graph_walker = BzrFetchGraphWalker(self.target, mapping)
 
211
        self.target.lock_write()
 
212
        try:
 
213
            self.target.start_write_group()
 
214
            try:
 
215
                import_git_objects(self.target, mapping,
 
216
                    iter(self.source.fetch_objects(determine_wants, graph_walker, 
 
217
                        progress)), pb)
 
218
            finally:
 
219
                self.target.commit_write_group()
 
220
        finally:
 
221
            self.target.unlock()
226
222
 
227
223
    @staticmethod
228
224
    def is_compatible(source, target):
229
225
        """Be compatible with GitRepository."""
230
226
        # FIXME: Also check target uses VersionedFile
231
 
        return (isinstance(source, GitRepository) and 
232
 
                target.supports_rich_root() and
233
 
                not isinstance(target, GitRepository))
234
 
 
235
 
 
236
 
class InterGitRepository(InterRepository):
237
 
 
238
 
    _matching_repo_format = GitFormat()
239
 
 
240
 
    @staticmethod
241
 
    def _get_repo_format_to_test():
242
 
        return None
243
 
 
244
 
    def copy_content(self, revision_id=None, pb=None):
245
 
        """See InterRepository.copy_content."""
246
 
        self.fetch(revision_id, pb, find_ghosts=False)
247
 
 
248
 
    def fetch(self, revision_id=None, pb=None, find_ghosts=False, 
249
 
              mapping=None):
250
 
        if mapping is None:
251
 
            mapping = self.source.get_mapping()
252
 
        def progress(text):
253
 
            info("git: %s", text)
254
 
        r = self.target._git
255
 
        if revision_id is None:
256
 
            determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
257
 
        else:
258
 
            args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
259
 
            determine_wants = lambda x: [y for y in args if not y in r.object_store]
260
 
 
261
 
        graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
262
 
        f, commit = r.object_store.add_pack()
263
 
        try:
264
 
            self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
265
 
            f.close()
266
 
            commit()
267
 
        except:
268
 
            f.close()
269
 
            raise
270
 
 
271
 
    @staticmethod
272
 
    def is_compatible(source, target):
273
 
        """Be compatible with GitRepository."""
274
 
        return (isinstance(source, GitRepository) and 
275
 
                isinstance(target, GitRepository))
 
227
        return (isinstance(source, LocalGitRepository) and 
 
228
                target.supports_rich_root())