/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5724.1.3 by John Arbash Meinel
Change the exporters to ensure that we are writing the data out in binary mode.
1
# Copyright (C) 2005, 2006, 2008-2011 Canonical Ltd
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
2
#
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
7
#
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
12
#
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
16
6379.6.7 by Jelmer Vernooij
Move importing from future until after doc string, otherwise the doc string will disappear.
17
"""Export a tree to a tarball."""
18
6379.6.3 by Jelmer Vernooij
Use absolute_import.
19
from __future__ import absolute_import
20
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
21
import os
3368.2.32 by Ian Clatworthy
add --filters to export command
22
import sys
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
23
import tarfile
3408.7.1 by Martin Pool
Support tarball export to stdout
24
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
25
from .. import (
5718.1.1 by Jelmer Vernooij
Fix import of BzrError.
26
    errors,
27
    osutils,
28
    )
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
29
from ..export import _export_iter_entries
30
from ..sixish import (
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
31
    BytesIO,
32
    )
5718.5.2 by Jelmer Vernooij
Factor out export_tarball.
33
5967.6.2 by Martin Pool
Delete fairly useless and repetitive per-format export single-call functions.
34
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
35
def prepare_tarball_item(tree, root, final_path, tree_path, entry, force_mtime=None):
5952.1.17 by geoffreyfishing at gmail
Renamed export item function in tar exporter.
36
    """Prepare a tarball item for exporting
5967.6.2 by Martin Pool
Delete fairly useless and repetitive per-format export single-call functions.
37
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
38
    :param tree: Tree to export
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
39
    :param final_path: Final path to place item
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
40
    :param tree_path: Path for the entry in the tree
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
41
    :param entry: Entry to export
5967.6.2 by Martin Pool
Delete fairly useless and repetitive per-format export single-call functions.
42
    :param force_mtime: Option mtime to force, instead of using tree
43
        timestamps.
44
5952.1.17 by geoffreyfishing at gmail
Renamed export item function in tar exporter.
45
    Returns a (tarinfo, fileobj) tuple
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
46
    """
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
47
    filename = osutils.pathjoin(root, final_path).encode('utf8')
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
48
    item = tarfile.TarInfo(filename)
49
    if force_mtime is not None:
50
        item.mtime = force_mtime
51
    else:
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
52
        item.mtime = tree.get_file_mtime(entry.file_id, tree_path)
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
53
    if entry.kind == "file":
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
54
        item.type = tarfile.REGTYPE
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
55
        if tree.is_executable(entry.file_id, tree_path):
6619.3.17 by Jelmer Vernooij
Run 2to3 numliterals fixer.
56
            item.mode = 0o755
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
57
        else:
6619.3.17 by Jelmer Vernooij
Run 2to3 numliterals fixer.
58
            item.mode = 0o644
6006.3.4 by Martin Pool
Support exporting tarballs from ContentFilterTree
59
        # This brings the whole file into memory, but that's almost needed for
60
        # the tarfile contract, which wants the size of the file up front.  We
61
        # want to make sure it doesn't change, and we need to read it in one
62
        # go for content filtering.
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
63
        content = tree.get_file_text(entry.file_id, tree_path)
6006.3.4 by Martin Pool
Support exporting tarballs from ContentFilterTree
64
        item.size = len(content)
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
65
        fileobj = BytesIO(content)
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
66
    elif entry.kind == "directory":
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
67
        item.type = tarfile.DIRTYPE
68
        item.name += '/'
69
        item.size = 0
6619.3.14 by Jelmer Vernooij
Convert some octal numbers to new notations.
70
        item.mode = 0o755
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
71
        fileobj = None
5952.1.14 by geoffreyfishing at gmail
Cleaned up code and removed unnecessary arguments.
72
    elif entry.kind == "symlink":
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
73
        item.type = tarfile.SYMTYPE
74
        item.size = 0
6619.3.14 by Jelmer Vernooij
Convert some octal numbers to new notations.
75
        item.mode = 0o755
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
76
        item.linkname = tree.get_symlink_target(entry.file_id, tree_path)
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
77
        fileobj = None
78
    else:
5952.1.26 by Vincent Ladeuil
Fix the test failure, streams should be closed. In the right order.
79
        raise errors.BzrError("don't know how to export {%s} of kind %r"
5952.1.18 by geoffreyfishing at gmail
Fixed line ending problems.
80
                              % (entry.file_id, entry.kind))
5952.1.4 by geoffreyfishing at gmail
Cleaned up export_tarball_item.
81
    return (item, fileobj)
1185.31.12 by John Arbash Meinel
Refactored the export code to make it easier to add new export formats.
82
5967.6.2 by Martin Pool
Delete fairly useless and repetitive per-format export single-call functions.
83
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
84
def export_tarball_generator(tree, ball, root, subdir=None, force_mtime=None):
5967.6.3 by Martin Pool
Further shrink export code
85
    """Export tree contents to a tarball.
86
87
    :returns: A generator that will repeatedly produce None as each file is
88
        emitted.  The entire generator must be consumed to complete writing
89
        the file.
5952.1.3 by geoffreyfishing at gmail
Updated code to requested format.
90
91
    :param tree: Tree to export
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
92
5967.6.3 by Martin Pool
Further shrink export code
93
    :param ball: Tarball to export to; it will be closed when writing is
94
        complete.
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
95
5952.1.3 by geoffreyfishing at gmail
Updated code to requested format.
96
    :param subdir: Sub directory to export
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
97
98
    :param force_mtime: Option mtime to force, instead of using tree
99
        timestamps.
5952.1.3 by geoffreyfishing at gmail
Updated code to requested format.
100
    """
5967.6.4 by Martin Pool
Close tarball from finally block (thanks jam)
101
    try:
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
102
        for final_path, tree_path, entry in _export_iter_entries(tree, subdir):
5967.6.4 by Martin Pool
Close tarball from finally block (thanks jam)
103
            (item, fileobj) = prepare_tarball_item(
6331.2.1 by Jelmer Vernooij
Consistently pass tree path when exporting.
104
                tree, root, final_path, tree_path, entry, force_mtime)
5967.6.4 by Martin Pool
Close tarball from finally block (thanks jam)
105
            ball.addfile(item, fileobj)
106
            yield
107
    finally:
108
        ball.close()
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
109
110
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
111
def tgz_exporter_generator(tree, dest, root, subdir, force_mtime=None,
112
    fileobj=None):
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
113
    """Export this tree to a new tar file.
114
115
    `dest` will be created holding the contents of this tree; if it
116
    already exists, it will be clobbered, like with "tar -c".
117
    """
118
    import gzip
5718.5.16 by Jelmer Vernooij
Use revision tree timestamp if possible.
119
    if force_mtime is not None:
120
        root_mtime = force_mtime
121
    elif (getattr(tree, "repository", None) and
122
          getattr(tree, "get_revision_id", None)):
123
        # If this is a revision tree, use the revisions' timestamp
124
        rev = tree.repository.get_revision(tree.get_revision_id())
125
        root_mtime = rev.timestamp
126
    elif tree.get_root_id() is not None:
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
127
        root_mtime = tree.get_file_mtime(tree.get_root_id())
5718.5.9 by Jelmer Vernooij
Add test for export zip to stdout.
128
    else:
5718.5.23 by Jelmer Vernooij
Leave default timestamp.
129
        root_mtime = None
5861.2.2 by Wouter van Heyst
Heavy handed approach to ensuring all streams are closed
130
131
    is_stdout = False
6018.1.1 by Martin Pool
Avoid unset local variable
132
    basename = None
5952.1.11 by geoffreyfishing at gmail
Implemented fileobj on tgz_exporter.
133
    if fileobj is not None:
134
        stream = fileobj
135
    elif dest == '-':
5718.5.22 by Jelmer Vernooij
Cope with mtime not always being available.
136
        stream = sys.stdout
5861.2.2 by Wouter van Heyst
Heavy handed approach to ensuring all streams are closed
137
        is_stdout = True
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
138
    else:
5724.1.4 by John Arbash Meinel
actually make the plain_tar export work again.
139
        stream = open(dest, 'wb')
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
140
        # gzip file is used with an explicit fileobj so that
141
        # the basename can be stored in the gzip file rather than
142
        # dest. (bug 102234)
5718.5.22 by Jelmer Vernooij
Cope with mtime not always being available.
143
        basename = os.path.basename(dest)
144
    try:
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
145
        zipstream = gzip.GzipFile(basename, 'w', fileobj=stream,
5952.1.18 by geoffreyfishing at gmail
Fixed line ending problems.
146
                                  mtime=root_mtime)
5718.5.22 by Jelmer Vernooij
Cope with mtime not always being available.
147
    except TypeError:
148
        # Python < 2.7 doesn't support the mtime argument
5861.2.2 by Wouter van Heyst
Heavy handed approach to ensuring all streams are closed
149
        zipstream = gzip.GzipFile(basename, 'w', fileobj=stream)
150
    ball = tarfile.open(None, 'w|', fileobj=zipstream)
5967.6.3 by Martin Pool
Further shrink export code
151
    for _ in export_tarball_generator(
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
152
        tree, ball, root, subdir, force_mtime):
5952.1.15 by geoffreyfishing at gmail
Major code cleanup.
153
        yield
5952.1.26 by Vincent Ladeuil
Fix the test failure, streams should be closed. In the right order.
154
    # Closing zipstream may trigger writes to stream
5952.1.15 by geoffreyfishing at gmail
Major code cleanup.
155
    zipstream.close()
156
    if not is_stdout:
5952.1.26 by Vincent Ladeuil
Fix the test failure, streams should be closed. In the right order.
157
        # Now we can safely close the stream
5952.1.15 by geoffreyfishing at gmail
Major code cleanup.
158
        stream.close()
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
159
160
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
161
def tbz_exporter_generator(tree, dest, root, subdir,
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
162
                           force_mtime=None, fileobj=None):
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
163
    """Export this tree to a new tar file.
164
165
    `dest` will be created holding the contents of this tree; if it
166
    already exists, it will be clobbered, like with "tar -c".
167
    """
5952.1.12 by geoffreyfishing at gmail
Implemented fileobj in tbz_exporter.
168
    if fileobj is not None:
169
        ball = tarfile.open(None, 'w|bz2', fileobj)
170
    elif dest == '-':
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
171
        ball = tarfile.open(None, 'w|bz2', sys.stdout)
172
    else:
5967.6.3 by Martin Pool
Further shrink export code
173
        # tarfile.open goes on to do 'os.getcwd() + dest' for opening the
174
        # tar file. With dest being unicode, this throws UnicodeDecodeError
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
175
        # unless we encode dest before passing it on. This works around
5967.6.3 by Martin Pool
Further shrink export code
176
        # upstream python bug http://bugs.python.org/issue8396 (fixed in
177
        # Python 2.6.5 and 2.7b1)
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
178
        ball = tarfile.open(dest.encode(osutils._fs_enc), 'w:bz2')
5967.6.3 by Martin Pool
Further shrink export code
179
    return export_tarball_generator(
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
180
        tree, ball, root, subdir, force_mtime)
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
181
182
183
def plain_tar_exporter_generator(tree, dest, root, subdir, compression=None,
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
184
    force_mtime=None, fileobj=None):
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
185
    """Export this tree to a new tar file.
186
187
    `dest` will be created holding the contents of this tree; if it
188
    already exists, it will be clobbered, like with "tar -c".
189
    """
5952.1.13 by geoffreyfishing at gmail
Adjusted .tar.lzma
190
    if fileobj is not None:
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
191
        stream = fileobj
5952.1.13 by geoffreyfishing at gmail
Adjusted .tar.lzma
192
    elif dest == '-':
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
193
        stream = sys.stdout
5718.5.4 by Jelmer Vernooij
fix timestamp in tgz files.
194
    else:
5724.1.3 by John Arbash Meinel
Change the exporters to ensure that we are writing the data out in binary mode.
195
        stream = open(dest, 'wb')
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
196
    ball = tarfile.open(None, 'w|', stream)
5967.6.3 by Martin Pool
Further shrink export code
197
    return export_tarball_generator(
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
198
        tree, ball, root, subdir, force_mtime)
199
200
201
def tar_xz_exporter_generator(tree, dest, root, subdir,
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
202
                              force_mtime=None, fileobj=None):
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
203
    return tar_lzma_exporter_generator(tree, dest, root, subdir,
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
204
                                       force_mtime, fileobj, "xz")
205
206
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
207
def tar_lzma_exporter_generator(tree, dest, root, subdir,
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
208
                      force_mtime=None, fileobj=None,
209
                                compression_format="alone"):
5718.5.17 by Jelmer Vernooij
Support tar.lzma.
210
    """Export this tree to a new .tar.lzma file.
5718.5.10 by Jelmer Vernooij
Support creating .tar.xz files.
211
212
    `dest` will be created holding the contents of this tree; if it
213
    already exists, it will be clobbered, like with "tar -c".
214
    """
5718.5.11 by Jelmer Vernooij
Tests, tests, tests.
215
    if dest == '-':
5718.5.17 by Jelmer Vernooij
Support tar.lzma.
216
        raise errors.BzrError("Writing to stdout not supported for .tar.lzma")
5718.5.11 by Jelmer Vernooij
Tests, tests, tests.
217
5952.1.13 by geoffreyfishing at gmail
Adjusted .tar.lzma
218
    if fileobj is not None:
5952.2.1 by Vincent Ladeuil
PEP8 tweaks, lines too long, spaces at end of lines.
219
        raise errors.BzrError(
220
            "Writing to fileobject not supported for .tar.lzma")
5718.5.10 by Jelmer Vernooij
Support creating .tar.xz files.
221
    try:
222
        import lzma
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
223
    except ImportError as e:
5718.5.10 by Jelmer Vernooij
Support creating .tar.xz files.
224
        raise errors.DependencyNotPresent('lzma', e)
225
5718.5.17 by Jelmer Vernooij
Support tar.lzma.
226
    stream = lzma.LZMAFile(dest.encode(osutils._fs_enc), 'w',
5967.6.3 by Martin Pool
Further shrink export code
227
        options={"format": compression_format})
5718.5.15 by Jelmer Vernooij
Only write out basename of the tarfile to the gzip file.
228
    ball = tarfile.open(None, 'w:', fileobj=stream)
5967.6.3 by Martin Pool
Further shrink export code
229
    return export_tarball_generator(
6006.3.7 by Martin Pool
Remove duplicated content-filtering code from exporters
230
        tree, ball, root, subdir, force_mtime=force_mtime)