/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/upstream_import.py

  • Committer: Breezy landing bot
  • Author(s): Jelmer Vernooij
  • Date: 2018-11-16 18:26:22 UTC
  • mfrom: (7167.1.4 run-flake8)
  • Revision ID: breezy.the.bot@gmail.com-20181116182622-qw3gan3hz78a2imw
Add a flake8 test.

Merged from https://code.launchpad.net/~jelmer/brz/run-flake8/+merge/358902

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/upstream_import.py
 
1
# Copyright (C) 2006-2012 Aaron Bentley
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
16
 
 
17
"""Import upstream source into a branch"""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
import errno
 
22
from io import (
 
23
    BytesIO,
 
24
    )
 
25
import os
 
26
import re
 
27
import stat
 
28
import tarfile
 
29
import zipfile
 
30
 
 
31
from . import generate_ids, urlutils
 
32
from .controldir import ControlDir, is_control_filename
 
33
from .errors import (BzrError, NoSuchFile, BzrCommandError, NotBranchError)
 
34
from .osutils import (pathjoin, isdir, file_iterator, basename,
 
35
                      file_kind, splitpath)
 
36
from .sixish import (
 
37
    text_type,
 
38
    )
 
39
from .trace import warning
 
40
from .transform import TreeTransform, resolve_conflicts, cook_conflicts
 
41
from .transport import get_transport
 
42
from .workingtree import WorkingTree
 
43
 
 
44
 
 
45
# TODO(jelmer): Move this to transport.py ?
 
46
def open_from_url(location):
 
47
    location = urlutils.normalize_url(location)
 
48
    dirname, basename = urlutils.split(location)
 
49
    if location.endswith('/') and not basename.endswith('/'):
 
50
        basename += '/'
 
51
    return get_transport(dirname).get(basename)
 
52
 
 
53
 
 
54
class NotArchiveType(BzrError):
 
55
 
 
56
    _fmt = '%(path)s is not an archive.'
 
57
 
 
58
    def __init__(self, path):
 
59
        BzrError.__init__(self)
 
60
        self.path = path
 
61
 
 
62
 
 
63
class ZipFileWrapper(object):
 
64
 
 
65
    def __init__(self, fileobj, mode):
 
66
        self.zipfile = zipfile.ZipFile(fileobj, mode)
 
67
 
 
68
    def getmembers(self):
 
69
        for info in self.zipfile.infolist():
 
70
            yield ZipInfoWrapper(self.zipfile, info)
 
71
 
 
72
    def extractfile(self, infowrapper):
 
73
        return BytesIO(self.zipfile.read(infowrapper.name))
 
74
 
 
75
    def add(self, filename):
 
76
        if isdir(filename):
 
77
            self.zipfile.writestr(filename+'/', '')
 
78
        else:
 
79
            self.zipfile.write(filename)
 
80
 
 
81
    def close(self):
 
82
        self.zipfile.close()
 
83
 
 
84
 
 
85
class ZipInfoWrapper(object):
 
86
 
 
87
    def __init__(self, zipfile, info):
 
88
        self.info = info
 
89
        self.type = None
 
90
        self.name = info.filename
 
91
        self.zipfile = zipfile
 
92
        self.mode = 0o666
 
93
 
 
94
    def isdir(self):
 
95
        # Really? Eeeew!
 
96
        return bool(self.name.endswith('/'))
 
97
 
 
98
    def isreg(self):
 
99
        # Really? Eeeew!
 
100
        return not self.isdir()
 
101
 
 
102
 
 
103
class DirWrapper(object):
 
104
 
 
105
    def __init__(self, fileobj, mode='r'):
 
106
        if mode != 'r':
 
107
            raise AssertionError(
 
108
                'only readonly supported')
 
109
        self.root = os.path.realpath(fileobj.read().decode('utf-8'))
 
110
 
 
111
    def __repr__(self):
 
112
        return 'DirWrapper(%r)' % self.root
 
113
 
 
114
    def getmembers(self, subdir=None):
 
115
        if subdir is not None:
 
116
            mydir = pathjoin(self.root, subdir)
 
117
        else:
 
118
            mydir = self.root
 
119
        for child in os.listdir(mydir):
 
120
            if subdir is not None:
 
121
                child = pathjoin(subdir, child)
 
122
            fi = FileInfo(self.root, child)
 
123
            yield fi
 
124
            if fi.isdir():
 
125
                for v in self.getmembers(child):
 
126
                    yield v
 
127
 
 
128
    def extractfile(self, member):
 
129
        return open(member.fullpath, 'rb')
 
130
 
 
131
 
 
132
class FileInfo(object):
 
133
 
 
134
    def __init__(self, root, filepath):
 
135
        self.fullpath = pathjoin(root, filepath)
 
136
        self.root = root
 
137
        if filepath != '':
 
138
            self.name = pathjoin(basename(root), filepath)
 
139
        else:
 
140
            print('root %r' % root)
 
141
            self.name = basename(root)
 
142
        self.type = None
 
143
        stat = os.lstat(self.fullpath)
 
144
        self.mode = stat.st_mode
 
145
        if self.isdir():
 
146
            self.name += '/'
 
147
 
 
148
    def __repr__(self):
 
149
        return 'FileInfo(%r)' % self.name
 
150
 
 
151
    def isreg(self):
 
152
        return stat.S_ISREG(self.mode)
 
153
 
 
154
    def isdir(self):
 
155
        return stat.S_ISDIR(self.mode)
 
156
 
 
157
    def issym(self):
 
158
        if stat.S_ISLNK(self.mode):
 
159
            self.linkname = os.readlink(self.fullpath)
 
160
            return True
 
161
        else:
 
162
            return False
 
163
 
 
164
 
 
165
def top_path(path):
 
166
    """Return the top directory given in a path."""
 
167
    components = splitpath(path)
 
168
    if len(components) > 0:
 
169
        return components[0]
 
170
    else:
 
171
        return ''
 
172
 
 
173
 
 
174
def common_directory(names):
 
175
    """Determine a single directory prefix from a list of names"""
 
176
    possible_prefix = None
 
177
    for name in names:
 
178
        name_top = top_path(name)
 
179
        if name_top == '':
 
180
            return None
 
181
        if possible_prefix is None:
 
182
            possible_prefix = name_top
 
183
        else:
 
184
            if name_top != possible_prefix:
 
185
                return None
 
186
    return possible_prefix
 
187
 
 
188
 
 
189
def do_directory(tt, trans_id, tree, relative_path, path):
 
190
    if isdir(path) and tree.is_versioned(relative_path):
 
191
        tt.cancel_deletion(trans_id)
 
192
    else:
 
193
        tt.create_directory(trans_id)
 
194
 
 
195
 
 
196
def add_implied_parents(implied_parents, path):
 
197
    """Update the set of implied parents from a path"""
 
198
    parent = os.path.dirname(path)
 
199
    if parent in implied_parents:
 
200
        return
 
201
    implied_parents.add(parent)
 
202
    add_implied_parents(implied_parents, parent)
 
203
 
 
204
 
 
205
def names_of_files(tar_file):
 
206
    for member in tar_file.getmembers():
 
207
        if member.type != "g":
 
208
            yield member.name
 
209
 
 
210
 
 
211
def should_ignore(relative_path):
 
212
    return is_control_filename(top_path(relative_path))
 
213
 
 
214
 
 
215
def import_tar(tree, tar_input):
 
216
    """Replace the contents of a working directory with tarfile contents.
 
217
    The tarfile may be a gzipped stream.  File ids will be updated.
 
218
    """
 
219
    tar_file = tarfile.open('lala', 'r', tar_input)
 
220
    import_archive(tree, tar_file)
 
221
 
 
222
def import_zip(tree, zip_input):
 
223
    zip_file = ZipFileWrapper(zip_input, 'r')
 
224
    import_archive(tree, zip_file)
 
225
 
 
226
 
 
227
def import_dir(tree, dir_input):
 
228
    dir_file = DirWrapper(dir_input)
 
229
    import_archive(tree, dir_file)
 
230
 
 
231
 
 
232
def import_archive(tree, archive_file):
 
233
    tt = TreeTransform(tree)
 
234
    try:
 
235
        import_archive_to_transform(tree, archive_file, tt)
 
236
        tt.apply()
 
237
    finally:
 
238
        tt.finalize()
 
239
 
 
240
 
 
241
def import_archive_to_transform(tree, archive_file, tt):
 
242
    prefix = common_directory(names_of_files(archive_file))
 
243
    removed = set()
 
244
    for path, entry in tree.iter_entries_by_dir():
 
245
        if entry.parent_id is None:
 
246
            continue
 
247
        trans_id = tt.trans_id_tree_path(path)
 
248
        tt.delete_contents(trans_id)
 
249
        removed.add(path)
 
250
 
 
251
    added = set()
 
252
    implied_parents = set()
 
253
    seen = set()
 
254
    for member in archive_file.getmembers():
 
255
        if member.type == 'g':
 
256
            # type 'g' is a header
 
257
            continue
 
258
        # Inverse functionality in bzr uses utf-8.  We could also
 
259
        # interpret relative to fs encoding, which would match native
 
260
        # behaviour better.
 
261
        relative_path = member.name
 
262
        if not isinstance(relative_path, text_type):
 
263
            relative_path = relative_path.decode('utf-8')
 
264
        if prefix is not None:
 
265
            relative_path = relative_path[len(prefix)+1:]
 
266
            relative_path = relative_path.rstrip('/')
 
267
        if relative_path == '':
 
268
            continue
 
269
        if should_ignore(relative_path):
 
270
            continue
 
271
        add_implied_parents(implied_parents, relative_path)
 
272
        trans_id = tt.trans_id_tree_path(relative_path)
 
273
        added.add(relative_path.rstrip('/'))
 
274
        path = tree.abspath(relative_path)
 
275
        if member.name in seen:
 
276
            if tt.final_kind(trans_id) == 'file':
 
277
                tt.set_executability(None, trans_id)
 
278
            tt.cancel_creation(trans_id)
 
279
        seen.add(member.name)
 
280
        if member.isreg():
 
281
            tt.create_file(file_iterator(archive_file.extractfile(member)),
 
282
                           trans_id)
 
283
            executable = (member.mode & 0o111) != 0
 
284
            tt.set_executability(executable, trans_id)
 
285
        elif member.isdir():
 
286
            do_directory(tt, trans_id, tree, relative_path, path)
 
287
        elif member.issym():
 
288
            tt.create_symlink(member.linkname, trans_id)
 
289
        else:
 
290
            continue
 
291
        if tt.tree_file_id(trans_id) is None:
 
292
            name = basename(member.name.rstrip('/'))
 
293
            file_id = generate_ids.gen_file_id(name)
 
294
            tt.version_file(file_id, trans_id)
 
295
 
 
296
    for relative_path in implied_parents.difference(added):
 
297
        if relative_path == "":
 
298
            continue
 
299
        trans_id = tt.trans_id_tree_path(relative_path)
 
300
        path = tree.abspath(relative_path)
 
301
        do_directory(tt, trans_id, tree, relative_path, path)
 
302
        if tt.tree_file_id(trans_id) is None:
 
303
            tt.version_file(trans_id, trans_id)
 
304
        added.add(relative_path)
 
305
 
 
306
    for path in removed.difference(added):
 
307
        tt.unversion_file(tt.trans_id_tree_path(path))
 
308
 
 
309
    for conflict in cook_conflicts(resolve_conflicts(tt), tt):
 
310
        warning(conflict)
 
311
 
 
312
 
 
313
def do_import(source, tree_directory=None):
 
314
    """Implementation of import command.  Intended for UI only"""
 
315
    if tree_directory is not None:
 
316
        try:
 
317
            tree = WorkingTree.open(tree_directory)
 
318
        except NotBranchError:
 
319
            if not os.path.exists(tree_directory):
 
320
                os.mkdir(tree_directory)
 
321
            branch = ControlDir.create_branch_convenience(tree_directory)
 
322
            tree = branch.controldir.open_workingtree()
 
323
    else:
 
324
        tree = WorkingTree.open_containing('.')[0]
 
325
    with tree.lock_write():
 
326
        if tree.changes_from(tree.basis_tree()).has_changed():
 
327
            raise BzrCommandError("Working tree has uncommitted changes.")
 
328
 
 
329
        try:
 
330
            archive, external_compressor = get_archive_type(source)
 
331
        except NotArchiveType:
 
332
            if file_kind(source) == 'directory':
 
333
                s = BytesIO(source.encode('utf-8'))
 
334
                s.seek(0)
 
335
                import_dir(tree, s)
 
336
            else:
 
337
                raise BzrCommandError('Unhandled import source')
 
338
        else:
 
339
            if archive == 'zip':
 
340
                import_zip(tree, open_from_url(source))
 
341
            elif archive == 'tar':
 
342
                try:
 
343
                    tar_input = open_from_url(source)
 
344
                    if external_compressor == 'bz2':
 
345
                        import bz2
 
346
                        tar_input = BytesIO(bz2.decompress(tar_input.read()))
 
347
                    elif external_compressor == 'lzma':
 
348
                        import lzma
 
349
                        tar_input = BytesIO(lzma.decompress(tar_input.read()))
 
350
                except IOError as e:
 
351
                    if e.errno == errno.ENOENT:
 
352
                        raise NoSuchFile(source)
 
353
                try:
 
354
                    import_tar(tree, tar_input)
 
355
                finally:
 
356
                    tar_input.close()
 
357
 
 
358
 
 
359
def get_archive_type(path):
 
360
    """Return the type of archive and compressor indicated by path name.
 
361
 
 
362
    Only external compressors are returned, so zip files are only
 
363
    ('zip', None).  .tgz is treated as ('tar', 'gz') and '.tar.xz' is treated
 
364
    as ('tar', 'lzma').
 
365
    """
 
366
    matches = re.match(r'.*\.(zip|tgz|tar(.(gz|bz2|lzma|xz))?)$', path)
 
367
    if not matches:
 
368
        raise NotArchiveType(path)
 
369
    external_compressor = None
 
370
    if matches.group(3) is not None:
 
371
        archive = 'tar'
 
372
        external_compressor = matches.group(3)
 
373
        if external_compressor == 'xz':
 
374
            external_compressor = 'lzma'
 
375
    elif matches.group(1) == 'tgz':
 
376
        return 'tar', 'gz'
 
377
    else:
 
378
        archive = matches.group(1)
 
379
    return archive, external_compressor