1
# Copyright (C) 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program. If not, see <http://www.gnu.org/licenses/>.
16
"""Fastimport/fastexport commands."""
18
from __future__ import absolute_import
20
from ... import controldir
21
from ...commands import Command
22
from ...option import Option, RegistryOption
23
from ...sixish import (
34
def _run(source, processor_factory, verbose=False, user_map=None, **kwargs):
35
"""Create and run a processor.
37
:param source: a filename or '-' for standard input. If the
38
filename ends in .gz, it will be opened as a gzip file and
39
the stream will be implicitly uncompressed
40
:param processor_factory: a callable for creating a processor
41
:param user_map: if not None, the file containing the user map.
43
from fastimport.errors import ParsingError
44
from ...errors import BzrCommandError
45
from fastimport import parser
46
stream = _get_source_stream(source)
47
user_mapper = _get_user_mapper(user_map)
48
proc = processor_factory(verbose=verbose, **kwargs)
49
p = parser.ImportParser(stream, verbose=verbose, user_mapper=user_mapper)
51
return proc.process(p.iter_commands)
52
except ParsingError as e:
53
raise BzrCommandError("%d: Parse error: %s" % (e.lineno, e))
56
def _get_source_stream(source):
57
if source == '-' or source is None:
59
stream = helpers.binary_stream(sys.stdin)
60
elif source.endswith('.gz'):
62
stream = gzip.open(source, "rb")
64
stream = open(source, "rb")
68
def _get_user_mapper(filename):
69
from . import user_mapper
75
return user_mapper.UserMapper(lines)
78
class cmd_fast_import(Command):
79
"""Backend for fast Bazaar data importers.
81
This command reads a mixed command/data stream and creates
82
branches in a Bazaar repository accordingly. The preferred
85
bzr fast-import project.fi project.bzr
87
Numerous commands are provided for generating a fast-import file
89
To specify standard input as the input stream, use a
90
source name of '-' (instead of project.fi). If the source name
91
ends in '.gz', it is assumed to be compressed in gzip format.
93
project.bzr will be created if it doesn't exist. If it exists
94
already, it should be empty or be an existing Bazaar repository
95
or branch. If not specified, the current directory is assumed.
97
fast-import will intelligently select the format to use when
98
creating a repository or branch. If you are running Bazaar 1.17
99
up to Bazaar 2.0, the default format for Bazaar 2.x ("2a") is used.
100
Otherwise, the current default format ("pack-0.92" for Bazaar 1.x)
101
is used. If you wish to specify a custom format, use the `--format`
106
To maintain backwards compatibility, fast-import lets you
107
create the target repository or standalone branch yourself.
108
It is recommended though that you let fast-import create
109
these for you instead.
111
:Branch mapping rules:
113
Git reference names are mapped to Bazaar branch names as follows:
115
* refs/heads/foo is mapped to foo
116
* refs/remotes/origin/foo is mapped to foo.remote
117
* refs/tags/foo is mapped to foo.tag
118
* */master is mapped to trunk, trunk.remote, etc.
119
* */trunk is mapped to git-trunk, git-trunk.remote, etc.
121
:Branch creation rules:
123
When a shared repository is created or found at the destination,
124
branches are created inside it. In the simple case of a single
125
branch (refs/heads/master) inside the input file, the branch is
128
When a standalone branch is found at the destination, the trunk
129
is imported there and warnings are output about any other branches
130
found in the input file.
132
When a branch in a shared repository is found at the destination,
133
that branch is made the trunk and other branches, if any, are
134
created in sister directories.
136
:Working tree updates:
138
The working tree is generated for the trunk branch. If multiple
139
branches are created, a message is output on completion explaining
140
how to create the working trees for other branches.
144
The fast-export-from-xxx commands typically call more advanced
145
xxx-fast-export scripts. You are welcome to use the advanced
146
scripts if you prefer.
148
If you wish to write a custom exporter for your project, see
149
http://bazaar-vcs.org/BzrFastImport for the detailed protocol
150
specification. In many cases, exporters can be written quite
151
quickly using whatever scripting/programming language you like.
155
Some source repositories store just the user name while Bazaar
156
prefers a full email address. You can adjust user-ids while
157
importing by using the --user-map option. The argument is a
158
text file with lines in the format::
162
Blank lines and lines beginning with # are ignored.
163
If old-id has the special value '@', then users without an
164
email address will get one created by using the matching new-id
165
as the domain, unless a more explicit address is given for them.
166
For example, given the user-map of::
169
bill = William Jones <bill@example.com>
171
then user-ids are mapped as follows::
173
maria => maria <maria@example.com>
174
bill => William Jones <bill@example.com>
178
User mapping is supported by both the fast-import and
179
fast-import-filter commands.
183
As some exporters (like git-fast-export) reuse blob data across
184
commits, fast-import makes two passes over the input file by
185
default. In the first pass, it collects data about what blobs are
186
used when, along with some other statistics (e.g. total number of
187
commits). In the second pass, it generates the repository and
192
The initial pass isn't done if the --info option is used
193
to explicitly pass in information about the input stream.
194
It also isn't done if the source is standard input. In the
195
latter case, memory consumption may be higher than otherwise
196
because some blobs may be kept in memory longer than necessary.
198
:Restarting an import:
200
At checkpoints and on completion, the commit-id -> revision-id
201
map is saved to a file called 'fastimport-id-map' in the control
202
directory for the repository (e.g. .bzr/repository). If the import
203
is interrupted or unexpectedly crashes, it can be started again
204
and this file will be used to skip over already loaded revisions.
205
As long as subsequent exports from the original source begin
206
with exactly the same revisions, you can use this feature to
207
maintain a mirror of a repository managed by a foreign tool.
208
If and when Bazaar is used to manage the repository, this file
209
can be safely deleted.
213
Import a Subversion repository into Bazaar::
215
svn-fast-export /svn/repo/path > project.fi
216
bzr fast-import project.fi project.bzr
218
Import a CVS repository into Bazaar::
220
cvs2git /cvs/repo/path > project.fi
221
bzr fast-import project.fi project.bzr
223
Import a Git repository into Bazaar::
226
git fast-export --all > project.fi
227
bzr fast-import project.fi project.bzr
229
Import a Mercurial repository into Bazaar::
232
hg fast-export > project.fi
233
bzr fast-import project.fi project.bzr
235
Import a Darcs repository into Bazaar::
238
darcs-fast-export > project.fi
239
bzr fast-import project.fi project.bzr
242
_see_also = ['fast-export', 'fast-import-filter', 'fast-import-info']
243
takes_args = ['source', 'destination?']
244
takes_options = ['verbose',
245
Option('user-map', type=text_type,
246
help="Path to file containing a map of user-ids.",
248
Option('info', type=text_type,
249
help="Path to file containing caching hints.",
252
help="Update all working trees, not just trunk's.",
254
Option('count', type=int,
255
help="Import this many revisions then exit.",
257
Option('checkpoint', type=int,
258
help="Checkpoint automatically every N revisions."
259
" The default is 10000.",
261
Option('autopack', type=int,
262
help="Pack every N checkpoints. The default is 4.",
264
Option('inv-cache', type=int,
265
help="Number of inventories to cache.",
267
RegistryOption.from_kwargs('mode',
268
'The import algorithm to use.',
269
title='Import Algorithm',
270
default='Use the preferred algorithm (inventory deltas).',
271
experimental="Enable experimental features.",
272
value_switches=True, enum_switch=False,
274
Option('import-marks', type=text_type,
275
help="Import marks from file."
277
Option('export-marks', type=text_type,
278
help="Export marks to file."
280
RegistryOption('format',
281
help='Specify a format for the created repository. See'
282
' "bzr help formats" for details.',
284
'breezy.controldir', 'format_registry'),
285
converter=lambda name: controldir.format_registry.make_controldir(
287
value_switches=False, title='Repository format'),
290
def run(self, source, destination='.', verbose=False, info=None,
291
trees=False, count=-1, checkpoint=10000, autopack=4, inv_cache=-1,
292
mode=None, import_marks=None, export_marks=None, format=None,
295
from .processors import generic_processor
296
from .helpers import (
297
open_destination_directory,
299
control = open_destination_directory(destination, format=format)
301
# If an information file was given and the source isn't stdin,
302
# generate the information by reading the source file as a first pass
303
if info is None and source != '-':
304
info = self._generate_info(source)
313
'checkpoint': checkpoint,
314
'autopack': autopack,
315
'inv-cache': inv_cache,
317
'import-marks': import_marks,
318
'export-marks': export_marks,
320
return _run(source, generic_processor.GenericProcessor,
321
bzrdir=control, params=params, verbose=verbose,
324
def _generate_info(self, source):
325
from ...sixish import StringIO
326
from fastimport import parser
327
from fastimport.errors import ParsingError
328
from ...errors import BzrCommandError
329
from fastimport.processors import info_processor
330
stream = _get_source_stream(source)
333
proc = info_processor.InfoProcessor(verbose=True, outf=output)
334
p = parser.ImportParser(stream)
336
return_code = proc.process(p.iter_commands)
337
except ParsingError as e:
338
raise BzrCommandError("%d: Parse error: %s" % (e.lineno, e))
339
lines = output.getvalue().splitlines()
346
class cmd_fast_export(Command):
347
"""Generate a fast-import stream from a Bazaar branch.
349
This program generates a stream from a Bazaar branch in fast-import
350
format used by tools such as bzr fast-import, git-fast-import and
353
It takes two optional arguments: the source bzr branch to export and
354
the destination to write the file to write the fastimport stream to.
356
If no source is specified, it will search for a branch in the
359
If no destination is given or the destination is '-', standard output
360
is used. Otherwise, the destination is the name of a file. If the
361
destination ends in '.gz', the output will be compressed into gzip
366
Recent versions of the fast-import specification support features
367
that allow effective round-tripping most of the metadata in Bazaar
368
branches. As such, fast-exporting a branch and fast-importing the data
369
produced will create a new repository with roughly equivalent history, i.e.
370
"bzr log -v -p --include-merges --forward" on the old branch and
371
new branch should produce similar, if not identical, results.
375
Be aware that the new repository may appear to have similar history
376
but internally it is quite different with new revision-ids and
377
file-ids assigned. As a consequence, the ability to easily merge
378
with branches based on the old repository is lost. Depending on your
379
reasons for producing a new repository, this may or may not be an
384
fast-export can use the following "extended features" to
385
produce a richer data stream:
387
* *multiple-authors* - if a commit has multiple authors (as commonly
388
occurs in pair-programming), all authors will be included in the
389
output, not just the first author
391
* *commit-properties* - custom metadata per commit that Bazaar stores
392
in revision properties (e.g. branch-nick and bugs fixed by this
393
change) will be included in the output.
395
* *empty-directories* - directories, even the empty ones, will be
396
included in the output.
398
To disable these features and produce output acceptable to git 1.6,
399
use the --plain option. To enable these features, use --no-plain.
400
Currently, --plain is the default but that will change in the near
401
future once the feature names and definitions are formally agreed
402
to by the broader fast-import developer community.
404
Git has stricter naming rules for tags and fast-export --plain
405
will skip tags which can't be imported into git. To replace characters
406
unsupported in git with an underscore instead, specify
411
It is sometimes convenient to simply truncate the revision history at a
412
certain point. The --baseline option, to be used in conjunction with -r,
413
emits a baseline commit containing the state of the entire source tree at
414
the first requested revision. This allows a user to produce a tree
415
identical to the original without munging multiple exports.
419
To produce data destined for import into Bazaar::
421
bzr fast-export --no-plain my-bzr-branch my.fi.gz
423
To produce data destined for Git 1.6::
425
bzr fast-export --plain my-bzr-branch my.fi
427
To import several unmerged but related branches into the same repository,
428
use the --{export,import}-marks options, and specify a name for the git
431
bzr fast-export --export-marks=marks.bzr project.dev |
432
GIT_DIR=project/.git git-fast-import --export-marks=marks.git
434
bzr fast-export --import-marks=marks.bzr -b other project.other |
435
GIT_DIR=project/.git git-fast-import --import-marks=marks.git
437
If you get a "Missing space after source" error from git-fast-import,
438
see the top of the commands.py module for a work-around.
440
Since bzr uses per-branch tags and git/hg use per-repo tags, the
441
way bzr fast-export presently emits tags (unconditional reset &
442
new ref) may result in clashes when several different branches
443
are imported into single git/hg repo. If this occurs, use the
444
bzr fast-export option --no-tags during the export of one or more
445
branches to avoid the issue.
448
_see_also = ['fast-import', 'fast-import-filter']
449
takes_args = ['source?', 'destination?']
450
takes_options = ['verbose', 'revision',
451
Option('git-branch', short_name='b', type=text_type,
453
help='Name of the git branch to create (default=master).'
455
Option('checkpoint', type=int, argname='N',
456
help="Checkpoint every N revisions (default=10000)."
458
Option('marks', type=text_type, argname='FILE',
459
help="Import marks from and export marks to file."
461
Option('import-marks', type=text_type, argname='FILE',
462
help="Import marks from file."
464
Option('export-marks', type=text_type, argname='FILE',
465
help="Export marks to file."
468
help="Exclude metadata to maximise interoperability."
470
Option('rewrite-tag-names',
471
help="Replace characters invalid in git with '_'"
472
" (plain mode only).",
475
help="Export an 'absolute' baseline commit prior to"
476
"the first relative commit",
479
help="Don't export tags"
482
encoding_type = 'exact'
484
def run(self, source=None, destination=None, verbose=False,
485
git_branch="master", checkpoint=10000, marks=None,
486
import_marks=None, export_marks=None, revision=None,
487
plain=True, rewrite_tag_names=False, no_tags=False, baseline=False):
489
from ...branch import Branch
490
from . import exporter
493
import_marks = export_marks = marks
498
branch = Branch.open_containing(source)[0]
499
outf = exporter._get_output_stream(destination)
500
exporter = exporter.BzrFastExporter(
502
outf=outf, ref=b"refs/heads/%s" % git_branch.encode('utf-8'),
503
checkpoint=checkpoint, import_marks_file=import_marks,
504
export_marks_file=export_marks, revision=revision, verbose=verbose,
505
plain_format=plain, rewrite_tags=rewrite_tag_names,
506
no_tags=no_tags, baseline=baseline)
507
return exporter.run()
added
removed
breezy/plugins/fastimport/cmds.py
