13
15
# You should have received a copy of the GNU General Public License
14
16
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""bzr upgrade logic."""
20
from bzrlib.bzrdir import BzrDir, format_registry
21
import bzrlib.errors as errors
22
from bzrlib.remote import RemoteBzrDir
23
import bzrlib.ui as ui
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
81
import hotshot, hotshot.stats
86
from bzrlib.branch import Branch, find_branch, BZR_BRANCH_FORMAT_5
87
from bzrlib.revfile import Revfile
88
from bzrlib.weave import Weave
89
from bzrlib.weavefile import read_weave, write_weave
90
from bzrlib.progress import ProgressBar
91
from bzrlib.atomicfile import AtomicFile
92
from bzrlib.xml4 import serializer_v4
93
from bzrlib.xml5 import serializer_v5
94
from bzrlib.trace import mutter, note, warning, enable_default_logging
95
from bzrlib.osutils import sha_strings, sha_string
96
from bzrlib.commit import merge_ancestry_lines
26
99
class Convert(object):
100
def __init__(self, base_dir):
102
self.converted_revs = set()
103
self.absent_revisions = set()
106
self.inventories = {}
28
def __init__(self, url, format=None):
30
self.bzrdir = BzrDir.open_unsupported(url)
31
# XXX: Change to cleanup
32
warning_id = 'cross_format_fetch'
33
saved_warning = warning_id in ui.ui_factory.suppressed_warnings
34
if isinstance(self.bzrdir, RemoteBzrDir):
35
self.bzrdir._ensure_real()
36
self.bzrdir = self.bzrdir._real_bzrdir
37
if self.bzrdir.root_transport.is_readonly():
38
raise errors.UpgradeReadonly
39
self.transport = self.bzrdir.root_transport
40
ui.ui_factory.suppressed_warnings.add(warning_id)
45
ui.ui_factory.suppressed_warnings.remove(warning_id)
47
110
def convert(self):
49
branch = self.bzrdir.open_branch()
50
if branch.user_url != self.bzrdir.user_url:
51
ui.ui_factory.note("This is a checkout. The branch (%s) needs to be "
52
"upgraded separately." %
55
except (errors.NotBranchError, errors.IncompatibleRepositories):
56
# might not be a format we can open without upgrading; see e.g.
57
# https://bugs.launchpad.net/bzr/+bug/253891
59
if self.format is None:
61
rich_root = self.bzrdir.find_repository()._format.rich_root_data
62
except errors.NoRepositoryPresent:
63
rich_root = False # assume no rich roots
65
format_name = "default-rich-root"
111
if not self._open_branch():
113
note('starting upgrade of %s', self.base)
114
self._backup_control_dir()
115
note('starting upgrade')
116
note('note: upgrade may be faster if all store files are ungzipped first')
117
self.pb = ProgressBar()
118
if not os.path.isdir(self.base + '/.bzr/weaves'):
119
os.mkdir(self.base + '/.bzr/weaves')
120
self.inv_weave = Weave('inventory')
121
self.anc_weave = Weave('ancestry')
123
# holds in-memory weaves for all files
124
self.text_weaves = {}
125
os.remove(self.branch.controlfilename('branch-format'))
126
self._convert_working_inv()
127
rev_history = self.branch.revision_history()
128
# to_read is a stack holding the revisions we still need to process;
129
# appending to it adds new highest-priority revisions
130
self.known_revisions = set(rev_history)
131
self.to_read = [rev_history[-1]]
133
rev_id = self.to_read.pop()
134
if (rev_id not in self.revisions
135
and rev_id not in self.absent_revisions):
136
self._load_one_rev(rev_id)
138
to_import = self._make_order()
139
for i, rev_id in enumerate(to_import):
140
self.pb.update('converting revision', i, len(to_import))
141
self._convert_one_rev(rev_id)
143
note('upgraded to weaves:')
144
note(' %6d revisions and inventories' % len(self.revisions))
145
note(' %6d absent revisions removed' % len(self.absent_revisions))
146
note(' %6d texts' % self.text_count)
147
self._write_all_weaves()
148
self._write_all_revs()
149
self._set_new_format()
150
self._cleanup_spare_files()
153
def _open_branch(self):
154
self.branch = Branch(self.base, relax_version_check=True)
155
if self.branch._branch_format == 5:
156
note('this branch is already in the most current format')
158
if self.branch._branch_format != 4:
159
raise BzrError("cannot upgrade from branch format %r" %
160
self.branch._branch_format)
164
def _set_new_format(self):
165
f = self.branch.controlfile('branch-format', 'wb')
167
f.write(BZR_BRANCH_FORMAT_5)
172
def _cleanup_spare_files(self):
173
for n in 'merged-patches', 'pending-merged-patches':
174
p = self.branch.controlfilename(n)
175
if not os.path.exists(p):
177
## assert os.path.getsize(p) == 0
179
shutil.rmtree(self.base + '/.bzr/inventory-store')
180
shutil.rmtree(self.base + '/.bzr/text-store')
183
def _backup_control_dir(self):
184
orig = self.base + '/.bzr'
185
backup = orig + '.backup'
186
shutil.copytree(orig, backup)
187
note('%s has been backed up to %s', orig, backup)
188
note('if conversion fails, you can move this directory back to .bzr')
189
note('if it succeeds, you can remove this directory if you wish')
192
def _convert_working_inv(self):
194
inv = serializer_v4.read_inventory(branch.controlfile('inventory', 'rb'))
195
serializer_v5.write_inventory(inv, branch.controlfile('inventory', 'wb'))
199
def _write_all_weaves(self):
200
write_a_weave(self.inv_weave, self.base + '/.bzr/inventory.weave')
201
write_a_weave(self.anc_weave, self.base + '/.bzr/ancestry.weave')
204
for file_id, file_weave in self.text_weaves.items():
205
self.pb.update('writing weave', i, len(self.text_weaves))
206
write_a_weave(file_weave, self.base + '/.bzr/weaves/%s.weave' % file_id)
212
def _write_all_revs(self):
213
"""Write all revisions out in new form."""
214
shutil.rmtree(self.base + '/.bzr/revision-store')
215
os.mkdir(self.base + '/.bzr/revision-store')
217
for i, rev_id in enumerate(self.converted_revs):
218
self.pb.update('write revision', i, len(self.converted_revs))
219
f = file(self.base + '/.bzr/revision-store/%s' % rev_id, 'wb')
221
serializer_v5.write_revision(self.revisions[rev_id], f)
228
def _load_one_rev(self, rev_id):
229
"""Load a revision object into memory.
231
Any parents not either loaded or abandoned get queued to be
233
self.pb.update('loading revision',
235
len(self.known_revisions))
236
if rev_id not in self.branch.revision_store:
238
note('revision {%s} not present in branch; '
239
'will not be converted',
241
self.absent_revisions.add(rev_id)
243
rev_xml = self.branch.revision_store[rev_id].read()
244
rev = serializer_v4.read_revision_from_string(rev_xml)
245
for parent_id in rev.parent_ids:
246
self.known_revisions.add(parent_id)
247
self.to_read.append(parent_id)
248
self.revisions[rev_id] = rev
249
old_inv_xml = self.branch.inventory_store[rev_id].read()
250
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
251
assert rev.inventory_sha1 == sha_string(old_inv_xml)
252
self.inventories[rev_id] = inv
255
def _convert_one_rev(self, rev_id):
256
"""Convert revision and all referenced objects to new format."""
257
rev = self.revisions[rev_id]
258
inv = self.inventories[rev_id]
259
for parent_id in rev.parent_ids[:]:
260
if parent_id in self.absent_revisions:
261
rev.parent_ids.remove(parent_id)
263
note('remove {%s} as parent of {%s}', parent_id, rev_id)
264
self._convert_revision_contents(rev, inv)
265
# the XML is now updated with text versions
266
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
267
new_inv_sha1 = sha_string(new_inv_xml)
268
self.inv_weave.add(rev_id, rev.parent_ids,
269
new_inv_xml.splitlines(True),
271
# TODO: Upgrade revision XML and write that out
272
rev.inventory_sha1 = new_inv_sha1
273
self._make_rev_ancestry(rev)
274
self.converted_revs.add(rev_id)
277
def _make_rev_ancestry(self, rev):
278
rev_id = rev.revision_id
279
for parent_id in rev.parent_ids:
280
assert parent_id in self.converted_revs
282
lines = list(self.anc_weave.mash_iter(rev.parent_ids))
285
lines.append(rev_id + '\n')
287
parent_ancestries = [self.ancestries[p] for p in rev.parent_ids]
288
new_lines = merge_ancestry_lines(rev_id, parent_ancestries)
289
assert set(lines) == set(new_lines)
290
self.ancestries[rev_id] = new_lines
291
self.anc_weave.add(rev_id, rev.parent_ids, lines)
294
def _convert_revision_contents(self, rev, inv):
295
"""Convert all the files within a revision.
297
Also upgrade the inventory to refer to the text revision ids."""
298
rev_id = rev.revision_id
299
mutter('converting texts of revision {%s}',
303
self._set_name_version(rev, ie)
304
if ie.kind != 'file':
306
self._convert_file_version(rev, ie)
309
def _set_name_version(self, rev, ie):
310
"""Set name version for a file.
312
Done in a slightly lazy way: if the file is renamed or in a merge revision
313
it gets a new version, otherwise the same as before.
316
if len(rev.parent_ids) != 1:
317
ie.name_version = rev.revision_id
319
old_inv = self.inventories[rev.parent_ids[0]]
320
if not old_inv.has_id(file_id):
321
ie.name_version = rev.revision_id
67
format_name = "default"
68
format = format_registry.make_bzrdir(format_name)
323
old_ie = old_inv[file_id]
324
if (old_ie.parent_id != ie.parent_id
325
or old_ie.name != ie.name):
326
ie.name_version = rev.revision_id
328
ie.name_version = old_ie.name_version
332
def _convert_file_version(self, rev, ie):
333
"""Convert one version of one file.
335
The file needs to be added into the weave if it is a merge
336
of >=2 parents or if it's changed from its parent.
339
rev_id = rev.revision_id
340
w = self.text_weaves.get(file_id)
343
self.text_weaves[file_id] = w
346
for parent_id in rev.parent_ids:
347
##if parent_id in self.absent_revisions:
349
assert parent_id in self.converted_revs, \
350
'parent {%s} not converted' % parent_id
351
parent_inv = self.inventories[parent_id]
352
if parent_inv.has_id(file_id):
353
parent_ie = parent_inv[file_id]
354
old_text_version = parent_ie.text_version
355
assert old_text_version in self.converted_revs
356
if old_text_version not in file_parents:
357
file_parents.append(old_text_version)
358
if parent_ie.text_sha1 != ie.text_sha1:
360
if len(file_parents) != 1 or text_changed:
361
file_lines = self.branch.text_store[ie.text_id].readlines()
362
assert sha_strings(file_lines) == ie.text_sha1
363
assert sum(map(len, file_lines)) == ie.text_size
364
w.add(rev_id, file_parents, file_lines, ie.text_sha1)
365
ie.text_version = rev_id
367
##mutter('import text {%s} of {%s}',
368
## ie.text_id, file_id)
71
if not self.bzrdir.needs_format_conversion(format):
72
raise errors.UpToDateFormat(self.bzrdir._format)
73
if not self.bzrdir.can_convert_format():
74
raise errors.BzrError("cannot upgrade from bzrdir format %s" %
76
self.bzrdir.check_conversion_target(format)
77
ui.ui_factory.note('starting upgrade of %s' % self.transport.base)
79
self.bzrdir.backup_bzrdir()
80
while self.bzrdir.needs_format_conversion(format):
81
converter = self.bzrdir._format.get_converter(format)
82
self.bzrdir = converter.convert(self.bzrdir, None)
83
ui.ui_factory.note("finished")
86
def upgrade(url, format=None):
87
"""Upgrade to format, or the default bzrdir format if not supplied."""
370
##mutter('text of {%s} unchanged from parent', file_id)
371
ie.text_version = file_parents[0]
376
def _make_order(self):
377
"""Return a suitable order for importing revisions.
379
The order must be such that an revision is imported after all
380
its (present) parents.
382
todo = set(self.revisions.keys())
383
done = self.absent_revisions.copy()
386
# scan through looking for a revision whose parents
388
for rev_id in sorted(list(todo)):
389
rev = self.revisions[rev_id]
390
parent_ids = set(rev.parent_ids)
391
if parent_ids.issubset(done):
392
# can take this one now
399
def write_a_weave(weave, filename):
400
inv_wf = file(filename, 'wb')
402
write_weave(weave, inv_wf)
407
def upgrade(base_dir):
added
removed
bzrlib/upgrade.py
