/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to mapping.py

Fix tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
mapping.py
18
18
 
19
19
"""Converters, etc for going between Bazaar and Git ids."""
20
20
 
21
 
import stat
22
 
 
23
 
from bzrlib import (
24
 
    errors,
25
 
    foreign,
26
 
    osutils,
27
 
    trace,
28
 
    urlutils,
29
 
    )
30
 
try:
31
 
    from bzrlib import bencode
32
 
except ImportError:
33
 
    from bzrlib.util import bencode
34
 
from bzrlib.inventory import (
35
 
    ROOT_ID,
36
 
    )
 
21
from bzrlib import errors, foreign, urlutils
 
22
from bzrlib.inventory import ROOT_ID
37
23
from bzrlib.foreign import (
38
 
    ForeignVcs, 
39
 
    VcsMappingRegistry, 
40
 
    ForeignRevision,
41
 
    )
42
 
 
43
 
DEFAULT_FILE_MODE = stat.S_IFREG | 0644
44
 
 
 
24
        ForeignVcs, 
 
25
        VcsMappingRegistry, 
 
26
        ForeignRevision,
 
27
        )
45
28
 
46
29
def escape_file_id(file_id):
47
30
    return file_id.replace('_', '__').replace(' ', '_s')
48
31
 
49
32
 
50
33
def unescape_file_id(file_id):
51
 
    ret = []
52
 
    i = 0
53
 
    while i < len(file_id):
54
 
        if file_id[i] != '_':
55
 
            ret.append(file_id[i])
56
 
        else:
57
 
            if file_id[i+1] == '_':
58
 
                ret.append("_")
59
 
            elif file_id[i+1] == 's':
60
 
                ret.append(" ")
61
 
            else:
62
 
                raise AssertionError("unknown escape character %s" % file_id[i+1])
63
 
            i += 1
64
 
        i += 1
65
 
    return "".join(ret)
66
 
 
67
 
 
68
 
def fix_person_identifier(text):
69
 
    if "<" in text and ">" in text:
70
 
        return text
71
 
    return "%s <%s>" % (text, text)
72
 
 
73
 
 
74
 
def warn_escaped(commit, num_escaped):
75
 
    trace.warning("Escaped %d XML-invalid characters in %s. Will be unable "
76
 
                  "to regenerate the SHA map.", num_escaped, commit)
77
 
 
78
 
 
79
 
def warn_unusual_mode(commit, path, mode):
80
 
    trace.mutter("Unusual file mode %o for %s in %s. Storing as revision property. ",
81
 
                 mode, path, commit)
82
 
 
83
 
 
84
 
def squash_revision(target_repo, rev):
85
 
    """Remove characters that can't be stored from a revision, if necessary.
86
 
    
87
 
    :param target_repo: Repository in which the revision will be stored
88
 
    :param rev: Revision object, will be modified in-place
89
 
    """
90
 
    if not getattr(target_repo._serializer, "squashes_xml_invalid_characters", True):
91
 
        return
92
 
    from bzrlib.xml_serializer import escape_invalid_chars
93
 
    rev.message, num_escaped = escape_invalid_chars(rev.message)
94
 
    if num_escaped:
95
 
        warn_escaped(rev.foreign_revid, num_escaped)
96
 
    if 'author' in rev.properties:
97
 
        rev.properties['author'], num_escaped = escape_invalid_chars(
98
 
            rev.properties['author'])
99
 
        if num_escaped:
100
 
            warn_escaped(rev.foreign_revid, num_escaped)
101
 
    rev.committer, num_escaped = escape_invalid_chars(rev.committer)
102
 
    if num_escaped:
103
 
        warn_escaped(rev.foreign_revid, num_escaped)
 
34
    return file_id.replace("_s", " ").replace("__", "_")
104
35
 
105
36
 
106
37
class BzrGitMapping(foreign.VcsMapping):
126
57
        return bzr_rev_id[len(cls.revid_prefix)+1:], cls()
127
58
 
128
59
    def generate_file_id(self, path):
129
 
        # Git paths are just bytestrings
130
 
        # We must just hope they are valid UTF-8..
131
60
        if path == "":
132
61
            return ROOT_ID
133
 
        return escape_file_id(path)
134
 
 
135
 
    def parse_file_id(self, file_id):
136
 
        if file_id == ROOT_ID:
137
 
            return ""
138
 
        return unescape_file_id(file_id)
139
 
 
140
 
    def import_unusual_file_modes(self, rev, unusual_file_modes):
141
 
        if unusual_file_modes:
142
 
            ret = [(name, unusual_file_modes[name]) for name in sorted(unusual_file_modes.keys())]
143
 
            rev.properties['file-modes'] = bencode.bencode(ret)
144
 
 
145
 
    def export_unusual_file_modes(self, rev):
146
 
        try:
147
 
            return dict([(self.generate_file_id(path), mode) for (path, mode) in bencode.bdecode(rev.properties['file-modes'])])
148
 
        except KeyError:
149
 
            return {}
 
62
        return escape_file_id(path.encode('utf-8'))
150
63
 
151
64
    def import_commit(self, commit):
152
65
        """Convert a git commit to a bzr revision.
161
74
        rev.committer = str(commit.committer).decode("utf-8", "replace")
162
75
        if commit.committer != commit.author:
163
76
            rev.properties['author'] = str(commit.author).decode("utf-8", "replace")
164
 
 
165
 
        if commit.commit_time != commit.author_time:
166
 
            rev.properties['author-timestamp'] = str(commit.author_time)
167
 
        if commit.commit_timezone != commit.author_timezone:
168
 
            rev.properties['author-timezone'] = "%d" % (commit.author_timezone, )
169
77
        rev.timestamp = commit.commit_time
170
 
        rev.timezone = commit.commit_timezone
 
78
        rev.timezone = 0
171
79
        return rev
172
80
 
173
81
 
175
83
    revid_prefix = 'git-v1'
176
84
    experimental = False
177
85
 
178
 
    def __str__(self):
179
 
        return self.revid_prefix
180
 
 
181
86
 
182
87
class BzrGitMappingExperimental(BzrGitMappingv1):
183
88
    revid_prefix = 'git-experimental'
185
90
 
186
91
 
187
92
class GitMappingRegistry(VcsMappingRegistry):
188
 
    """Registry with available git mappings."""
189
93
 
190
94
    def revision_id_bzr_to_foreign(self, bzr_revid):
191
95
        if not bzr_revid.startswith("git-"):
205
109
 
206
110
 
207
111
class ForeignGit(ForeignVcs):
208
 
    """The Git Stupid Content Tracker"""
 
112
    """Foreign Git."""
209
113
 
210
114
    def __init__(self):
211
115
        super(ForeignGit, self).__init__(mapping_registry)
219
123
default_mapping = BzrGitMappingv1()
220
124
 
221
125
 
222
 
def text_to_blob(texts, entry):
223
 
    from dulwich.objects import Blob
224
 
    text = texts.get_record_stream([(entry.file_id, entry.revision)], 'unordered', True).next().get_bytes_as('fulltext')
225
 
    blob = Blob()
226
 
    blob._text = text
227
 
    return blob
228
 
 
229
 
 
230
 
def symlink_to_blob(entry):
231
 
    from dulwich.objects import Blob
232
 
    blob = Blob()
233
 
    blob._text = entry.symlink_target
234
 
    return blob
235
 
 
236
 
 
237
 
def mode_is_executable(mode):
238
 
    """Check if mode should be considered executable."""
239
 
    return bool(mode & 0111)
240
 
 
241
 
 
242
 
def mode_kind(mode):
243
 
    """Determine the Bazaar inventory kind based on Unix file mode."""
244
 
    entry_kind = (mode & 0700000) / 0100000
245
 
    if entry_kind == 0:
246
 
        return 'directory'
247
 
    elif entry_kind == 1:
248
 
        file_kind = (mode & 070000) / 010000
249
 
        if file_kind == 0:
250
 
            return 'file'
251
 
        elif file_kind == 2:
252
 
            return 'symlink'
253
 
        elif file_kind == 6:
254
 
            return 'tree-reference'
255
 
        else:
256
 
            raise AssertionError(
257
 
                "Unknown file kind %d, perms=%o." % (file_kind, mode,))
258
 
    else:
259
 
        raise AssertionError(
260
 
            "Unknown kind, perms=%r." % (mode,))
261
 
 
262
 
 
263
 
def entry_mode(entry):
264
 
    """Determine the git file mode for an inventory entry."""
265
 
    if entry.kind == 'directory':
266
 
        return stat.S_IFDIR
267
 
    elif entry.kind == 'symlink':
268
 
        return stat.S_IFLNK
269
 
    elif entry.kind == 'file':
270
 
        mode = stat.S_IFREG | 0644
271
 
        if entry.executable:
272
 
            mode |= 0111
273
 
        return mode
274
 
    else:
275
 
        raise AssertionError
276
 
 
277
 
 
278
 
def directory_to_tree(entry, lookup_ie_sha1, unusual_modes):
279
 
    from dulwich.objects import Tree
280
 
    tree = Tree()
281
 
    for name in sorted(entry.children.keys()):
282
 
        ie = entry.children[name]
283
 
        try:
284
 
            mode = unusual_modes[ie.file_id]
285
 
        except KeyError:
286
 
            mode = entry_mode(ie)
287
 
        tree.add(mode, name.encode("utf-8"), lookup_ie_sha1(ie))
288
 
    tree.serialize()
289
 
    return tree
290
 
 
291
 
 
292
 
def extract_unusual_modes(rev):
293
 
    try:
294
 
        foreign_revid, mapping = mapping_registry.parse_revision_id(rev.revision_id)
295
 
    except errors.InvalidRevisionId:
296
 
        return {}
297
 
    else:
298
 
        return mapping.export_unusual_file_modes(rev)
299
 
 
300
 
 
301
 
def inventory_to_tree_and_blobs(inventory, texts, mapping, unusual_modes, cur=None):
302
 
    """Convert a Bazaar tree to a Git tree.
303
 
 
304
 
    :return: Yields tuples with object sha1, object and path
305
 
    """
306
 
    from dulwich.objects import Tree
 
126
def inventory_to_tree_and_blobs(repo, mapping, revision_id):
 
127
    from dulwich.objects import Tree, Blob
 
128
    from bzrlib.inventory import InventoryDirectory, InventoryFile
307
129
    import stat
308
130
    stack = []
309
 
    if cur is None:
310
 
        cur = ""
 
131
    cur = ""
311
132
    tree = Tree()
312
133
 
 
134
    inv = repo.get_inventory(revision_id)
 
135
 
313
136
    # stack contains the set of trees that we haven't 
314
137
    # finished constructing
315
 
    for path, entry in inventory.iter_entries():
316
 
        while stack and not path.startswith(osutils.pathjoin(cur, "")):
317
 
            # We've hit a file that's not a child of the previous path
 
138
 
 
139
    for path, entry in inv.iter_entries():
 
140
        while stack and not path.startswith(cur):
318
141
            tree.serialize()
319
 
            sha = tree.id
320
 
            yield sha, tree, cur.encode("utf-8")
321
 
            mode = unusual_modes.get(cur.encode("utf-8"), stat.S_IFDIR)
322
 
            t = (mode, urlutils.basename(cur).encode('UTF-8'), sha)
 
142
            sha = tree.sha().hexdigest()
 
143
            yield sha, tree, cur
 
144
            t = (stat.S_IFDIR, urlutils.basename(cur).encode('UTF-8'), sha)
323
145
            cur, tree = stack.pop()
324
146
            tree.add(*t)
325
147
 
326
 
        if entry.kind == "directory":
 
148
        if type(entry) == InventoryDirectory:
327
149
            stack.append((cur, tree))
328
150
            cur = path
329
151
            tree = Tree()
330
 
        else:
331
 
            if entry.kind == "file":
332
 
                blob = text_to_blob(texts, entry)
333
 
            elif entry.kind == "symlink":
334
 
                blob = symlink_to_blob(entry)
335
 
            else:
336
 
                raise AssertionError("Unknown kind %s" % entry.kind)
337
 
            sha = blob.id
338
 
            yield sha, blob, path.encode("utf-8")
 
152
 
 
153
        if type(entry) == InventoryFile:
 
154
            #FIXME: We can make potentially make this Lazy to avoid shaing lots of stuff
 
155
            # and having all these objects in memory at once
 
156
            blob = Blob()
 
157
            _, blob._text = repo.iter_files_bytes([(entry.file_id, entry.revision, path)]).next()
 
158
            sha = blob.sha().hexdigest()
 
159
            yield sha, blob, path
 
160
 
339
161
            name = urlutils.basename(path).encode("utf-8")
340
 
            mode = unusual_modes.get(path.encode("utf-8"), entry_mode(entry))
 
162
            mode = stat.S_IFREG | 0644
 
163
            if entry.executable:
 
164
                mode |= 0111
341
165
            tree.add(mode, name, sha)
342
166
 
343
167
    while len(stack) > 1:
344
168
        tree.serialize()
345
 
        sha = tree.id
346
 
        yield sha, tree, cur.encode("utf-8")
347
 
        mode = unusual_modes.get(cur.encode('utf-8'), stat.S_IFDIR)
348
 
        t = (mode, urlutils.basename(cur).encode('UTF-8'), sha)
 
169
        sha = tree.sha().hexdigest()
 
170
        yield sha, tree, cur
 
171
        t = (stat.S_IFDIR, urlutils.basename(cur).encode('UTF-8'), sha)
349
172
        cur, tree = stack.pop()
350
173
        tree.add(*t)
351
174
 
352
175
    tree.serialize()
353
 
    yield tree.id, tree, cur.encode("utf-8")
 
176
    yield tree.sha().hexdigest(), tree, cur
354
177
 
355
178
 
356
179
def revision_to_commit(rev, tree_sha, parent_lookup):
362
185
    """
363
186
    from dulwich.objects import Commit
364
187
    commit = Commit()
365
 
    commit.tree = tree_sha
 
188
    commit._tree = tree_sha
366
189
    for p in rev.parent_ids:
367
190
        git_p = parent_lookup(p)
368
191
        if git_p is not None:
369
 
            assert len(git_p) == 40, "unexpected length for %r" % git_p
370
 
            commit.parents.append(git_p)
371
 
    commit.message = rev.message.encode("utf-8")
372
 
    commit.committer = fix_person_identifier(rev.committer.encode("utf-8"))
373
 
    commit.author = fix_person_identifier(rev.get_apparent_authors()[0].encode("utf-8"))
374
 
    commit.commit_time = long(rev.timestamp)
375
 
    if 'author-timestamp' in rev.properties:
376
 
        commit.author_time = long(rev.properties['author-timestamp'])
377
 
    else:
378
 
        commit.author_time = commit.commit_time
379
 
    commit.commit_timezone = rev.timezone
380
 
    if 'author-timezone' in rev.properties:
381
 
        commit.author_timezone = int(rev.properties['author-timezone'])
382
 
    else:
383
 
        commit.author_timezone = commit.commit_timezone 
 
192
            commit._parents.append(git_p)
 
193
    commit._message = rev.message.encode("utf-8")
 
194
    commit._committer = rev.committer.encode("utf-8")
 
195
    commit._author = rev.get_apparent_authors()[0].encode("utf-8")
 
196
    commit._commit_time = long(rev.timestamp)
 
197
    commit.serialize()
384
198
    return commit