/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/upstream_import.py

  • Committer: Breezy landing bot
  • Author(s): Colin Watson
  • Date: 2020-11-16 21:47:08 UTC
  • mfrom: (7521.1.1 remove-lp-workaround)
  • Revision ID: breezy.the.bot@gmail.com-20201116214708-jos209mgxi41oy15
Remove breezy.git workaround for bazaar.launchpad.net.

Merged from https://code.launchpad.net/~cjwatson/brz/remove-lp-workaround/+merge/393710

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/upstream_import.py
 
1
# Copyright (C) 2006-2012 Aaron Bentley
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
16
 
 
17
"""Import upstream source into a branch"""
 
18
 
 
19
import errno
 
20
from io import (
 
21
    BytesIO,
 
22
    )
 
23
import os
 
24
import re
 
25
import stat
 
26
import tarfile
 
27
import zipfile
 
28
 
 
29
from . import urlutils
 
30
from .bzr import generate_ids
 
31
from .controldir import ControlDir, is_control_filename
 
32
from .errors import (BzrError, NoSuchFile, CommandError, NotBranchError)
 
33
from .osutils import (pathjoin, isdir, file_iterator, basename,
 
34
                      file_kind, splitpath)
 
35
from .trace import warning
 
36
from .transform import resolve_conflicts
 
37
from .transport import get_transport
 
38
from .workingtree import WorkingTree
 
39
 
 
40
 
 
41
# TODO(jelmer): Move this to transport.py ?
 
42
def open_from_url(location):
 
43
    location = urlutils.normalize_url(location)
 
44
    dirname, basename = urlutils.split(location)
 
45
    if location.endswith('/') and not basename.endswith('/'):
 
46
        basename += '/'
 
47
    return get_transport(dirname).get(basename)
 
48
 
 
49
 
 
50
class NotArchiveType(BzrError):
 
51
 
 
52
    _fmt = '%(path)s is not an archive.'
 
53
 
 
54
    def __init__(self, path):
 
55
        BzrError.__init__(self)
 
56
        self.path = path
 
57
 
 
58
 
 
59
class ZipFileWrapper(object):
 
60
 
 
61
    def __init__(self, fileobj, mode):
 
62
        self.zipfile = zipfile.ZipFile(fileobj, mode)
 
63
 
 
64
    def getmembers(self):
 
65
        for info in self.zipfile.infolist():
 
66
            yield ZipInfoWrapper(self.zipfile, info)
 
67
 
 
68
    def extractfile(self, infowrapper):
 
69
        return BytesIO(self.zipfile.read(infowrapper.name))
 
70
 
 
71
    def add(self, filename):
 
72
        if isdir(filename):
 
73
            self.zipfile.writestr(filename + '/', '')
 
74
        else:
 
75
            self.zipfile.write(filename)
 
76
 
 
77
    def close(self):
 
78
        self.zipfile.close()
 
79
 
 
80
 
 
81
class ZipInfoWrapper(object):
 
82
 
 
83
    def __init__(self, zipfile, info):
 
84
        self.info = info
 
85
        self.type = None
 
86
        self.name = info.filename
 
87
        self.zipfile = zipfile
 
88
        self.mode = 0o666
 
89
 
 
90
    def isdir(self):
 
91
        # Really? Eeeew!
 
92
        return bool(self.name.endswith('/'))
 
93
 
 
94
    def isreg(self):
 
95
        # Really? Eeeew!
 
96
        return not self.isdir()
 
97
 
 
98
 
 
99
class DirWrapper(object):
 
100
 
 
101
    def __init__(self, fileobj, mode='r'):
 
102
        if mode != 'r':
 
103
            raise AssertionError(
 
104
                'only readonly supported')
 
105
        self.root = os.path.realpath(fileobj.read().decode('utf-8'))
 
106
 
 
107
    def __repr__(self):
 
108
        return 'DirWrapper(%r)' % self.root
 
109
 
 
110
    def getmembers(self, subdir=None):
 
111
        if subdir is not None:
 
112
            mydir = pathjoin(self.root, subdir)
 
113
        else:
 
114
            mydir = self.root
 
115
        for child in os.listdir(mydir):
 
116
            if subdir is not None:
 
117
                child = pathjoin(subdir, child)
 
118
            fi = FileInfo(self.root, child)
 
119
            yield fi
 
120
            if fi.isdir():
 
121
                for v in self.getmembers(child):
 
122
                    yield v
 
123
 
 
124
    def extractfile(self, member):
 
125
        return open(member.fullpath, 'rb')
 
126
 
 
127
 
 
128
class FileInfo(object):
 
129
 
 
130
    def __init__(self, root, filepath):
 
131
        self.fullpath = pathjoin(root, filepath)
 
132
        self.root = root
 
133
        if filepath != '':
 
134
            self.name = pathjoin(basename(root), filepath)
 
135
        else:
 
136
            print('root %r' % root)
 
137
            self.name = basename(root)
 
138
        self.type = None
 
139
        stat = os.lstat(self.fullpath)
 
140
        self.mode = stat.st_mode
 
141
        if self.isdir():
 
142
            self.name += '/'
 
143
 
 
144
    def __repr__(self):
 
145
        return 'FileInfo(%r)' % self.name
 
146
 
 
147
    def isreg(self):
 
148
        return stat.S_ISREG(self.mode)
 
149
 
 
150
    def isdir(self):
 
151
        return stat.S_ISDIR(self.mode)
 
152
 
 
153
    def issym(self):
 
154
        if stat.S_ISLNK(self.mode):
 
155
            self.linkname = os.readlink(self.fullpath)
 
156
            return True
 
157
        else:
 
158
            return False
 
159
 
 
160
 
 
161
def top_path(path):
 
162
    """Return the top directory given in a path."""
 
163
    components = splitpath(path)
 
164
    if len(components) > 0:
 
165
        return components[0]
 
166
    else:
 
167
        return ''
 
168
 
 
169
 
 
170
def common_directory(names):
 
171
    """Determine a single directory prefix from a list of names"""
 
172
    possible_prefix = None
 
173
    for name in names:
 
174
        name_top = top_path(name)
 
175
        if name_top == '':
 
176
            return None
 
177
        if possible_prefix is None:
 
178
            possible_prefix = name_top
 
179
        else:
 
180
            if name_top != possible_prefix:
 
181
                return None
 
182
    return possible_prefix
 
183
 
 
184
 
 
185
def do_directory(tt, trans_id, tree, relative_path, path):
 
186
    if isdir(path) and tree.is_versioned(relative_path):
 
187
        tt.cancel_deletion(trans_id)
 
188
    else:
 
189
        tt.create_directory(trans_id)
 
190
 
 
191
 
 
192
def add_implied_parents(implied_parents, path):
 
193
    """Update the set of implied parents from a path"""
 
194
    parent = os.path.dirname(path)
 
195
    if parent in implied_parents:
 
196
        return
 
197
    implied_parents.add(parent)
 
198
    add_implied_parents(implied_parents, parent)
 
199
 
 
200
 
 
201
def names_of_files(tar_file):
 
202
    for member in tar_file.getmembers():
 
203
        if member.type != "g":
 
204
            yield member.name
 
205
 
 
206
 
 
207
def should_ignore(relative_path):
 
208
    return is_control_filename(top_path(relative_path))
 
209
 
 
210
 
 
211
def import_tar(tree, tar_input):
 
212
    """Replace the contents of a working directory with tarfile contents.
 
213
    The tarfile may be a gzipped stream.  File ids will be updated.
 
214
    """
 
215
    tar_file = tarfile.open('lala', 'r', tar_input)
 
216
    import_archive(tree, tar_file)
 
217
 
 
218
 
 
219
def import_zip(tree, zip_input):
 
220
    zip_file = ZipFileWrapper(zip_input, 'r')
 
221
    import_archive(tree, zip_file)
 
222
 
 
223
 
 
224
def import_dir(tree, dir_input):
 
225
    dir_file = DirWrapper(dir_input)
 
226
    import_archive(tree, dir_file)
 
227
 
 
228
 
 
229
def import_archive(tree, archive_file):
 
230
    with tree.transform() as tt:
 
231
        import_archive_to_transform(tree, archive_file, tt)
 
232
        tt.apply()
 
233
 
 
234
 
 
235
def import_archive_to_transform(tree, archive_file, tt):
 
236
    prefix = common_directory(names_of_files(archive_file))
 
237
    removed = set()
 
238
    for path, entry in tree.iter_entries_by_dir():
 
239
        if entry.parent_id is None:
 
240
            continue
 
241
        trans_id = tt.trans_id_tree_path(path)
 
242
        tt.delete_contents(trans_id)
 
243
        removed.add(path)
 
244
 
 
245
    added = set()
 
246
    implied_parents = set()
 
247
    seen = set()
 
248
    for member in archive_file.getmembers():
 
249
        if member.type == 'g':
 
250
            # type 'g' is a header
 
251
            continue
 
252
        # Inverse functionality in bzr uses utf-8.  We could also
 
253
        # interpret relative to fs encoding, which would match native
 
254
        # behaviour better.
 
255
        relative_path = member.name
 
256
        if not isinstance(relative_path, str):
 
257
            relative_path = relative_path.decode('utf-8')
 
258
        if prefix is not None:
 
259
            relative_path = relative_path[len(prefix) + 1:]
 
260
            relative_path = relative_path.rstrip('/')
 
261
        if relative_path == '':
 
262
            continue
 
263
        if should_ignore(relative_path):
 
264
            continue
 
265
        add_implied_parents(implied_parents, relative_path)
 
266
        trans_id = tt.trans_id_tree_path(relative_path)
 
267
        added.add(relative_path.rstrip('/'))
 
268
        path = tree.abspath(relative_path)
 
269
        if member.name in seen:
 
270
            if tt.final_kind(trans_id) == 'file':
 
271
                tt.set_executability(None, trans_id)
 
272
            tt.cancel_creation(trans_id)
 
273
        seen.add(member.name)
 
274
        if member.isreg():
 
275
            tt.create_file(file_iterator(archive_file.extractfile(member)),
 
276
                           trans_id)
 
277
            executable = (member.mode & 0o111) != 0
 
278
            tt.set_executability(executable, trans_id)
 
279
        elif member.isdir():
 
280
            do_directory(tt, trans_id, tree, relative_path, path)
 
281
        elif member.issym():
 
282
            tt.create_symlink(member.linkname, trans_id)
 
283
        else:
 
284
            continue
 
285
        if tt.tree_file_id(trans_id) is None:
 
286
            name = basename(member.name.rstrip('/'))
 
287
            file_id = generate_ids.gen_file_id(name)
 
288
            tt.version_file(trans_id, file_id=file_id)
 
289
 
 
290
    for relative_path in implied_parents.difference(added):
 
291
        if relative_path == "":
 
292
            continue
 
293
        trans_id = tt.trans_id_tree_path(relative_path)
 
294
        path = tree.abspath(relative_path)
 
295
        do_directory(tt, trans_id, tree, relative_path, path)
 
296
        if tt.tree_file_id(trans_id) is None:
 
297
            tt.version_file(trans_id, file_id=trans_id)
 
298
        added.add(relative_path)
 
299
 
 
300
    for path in removed.difference(added):
 
301
        tt.unversion_file(tt.trans_id_tree_path(path))
 
302
 
 
303
    for conflict in tt.cook_conflicts(resolve_conflicts(tt)):
 
304
        warning(conflict)
 
305
 
 
306
 
 
307
def do_import(source, tree_directory=None):
 
308
    """Implementation of import command.  Intended for UI only"""
 
309
    if tree_directory is not None:
 
310
        try:
 
311
            tree = WorkingTree.open(tree_directory)
 
312
        except NotBranchError:
 
313
            if not os.path.exists(tree_directory):
 
314
                os.mkdir(tree_directory)
 
315
            branch = ControlDir.create_branch_convenience(tree_directory)
 
316
            tree = branch.controldir.open_workingtree()
 
317
    else:
 
318
        tree = WorkingTree.open_containing('.')[0]
 
319
    with tree.lock_write():
 
320
        if tree.changes_from(tree.basis_tree()).has_changed():
 
321
            raise CommandError("Working tree has uncommitted changes.")
 
322
 
 
323
        try:
 
324
            archive, external_compressor = get_archive_type(source)
 
325
        except NotArchiveType:
 
326
            if file_kind(source) == 'directory':
 
327
                s = BytesIO(source.encode('utf-8'))
 
328
                s.seek(0)
 
329
                import_dir(tree, s)
 
330
            else:
 
331
                raise CommandError('Unhandled import source')
 
332
        else:
 
333
            if archive == 'zip':
 
334
                import_zip(tree, open_from_url(source))
 
335
            elif archive == 'tar':
 
336
                try:
 
337
                    tar_input = open_from_url(source)
 
338
                    if external_compressor == 'bz2':
 
339
                        import bz2
 
340
                        tar_input = BytesIO(bz2.decompress(tar_input.read()))
 
341
                    elif external_compressor == 'lzma':
 
342
                        import lzma
 
343
                        tar_input = BytesIO(lzma.decompress(tar_input.read()))
 
344
                except IOError as e:
 
345
                    if e.errno == errno.ENOENT:
 
346
                        raise NoSuchFile(source)
 
347
                try:
 
348
                    import_tar(tree, tar_input)
 
349
                finally:
 
350
                    tar_input.close()
 
351
 
 
352
 
 
353
def get_archive_type(path):
 
354
    """Return the type of archive and compressor indicated by path name.
 
355
 
 
356
    Only external compressors are returned, so zip files are only
 
357
    ('zip', None).  .tgz is treated as ('tar', 'gz') and '.tar.xz' is treated
 
358
    as ('tar', 'lzma').
 
359
    """
 
360
    matches = re.match(r'.*\.(zip|tgz|tar(.(gz|bz2|lzma|xz))?)$', path)
 
361
    if not matches:
 
362
        raise NotArchiveType(path)
 
363
    external_compressor = None
 
364
    if matches.group(3) is not None:
 
365
        archive = 'tar'
 
366
        external_compressor = matches.group(3)
 
367
        if external_compressor == 'xz':
 
368
            external_compressor = 'lzma'
 
369
    elif matches.group(1) == 'tgz':
 
370
        return 'tar', 'gz'
 
371
    else:
 
372
        archive = matches.group(1)
 
373
    return archive, external_compressor