3
# Copyright (C) 2005 Canonical Ltd
5
# This program is free software; you can redistribute it and/or modify
6
# it under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 2 of the License, or
8
# (at your option) any later version.
10
# This program is distributed in the hope that it will be useful,
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
15
# You should have received a copy of the GNU General Public License
16
# along with this program; if not, write to the Free Software
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
"""Experiment in converting existing bzr branches to weaves."""
21
# To make this properly useful
23
# 1. assign text version ids, and put those text versions into
24
# the inventory as they're converted.
26
# 2. keep track of the previous version of each file, rather than
27
# just using the last one imported
29
# 3. assign entry versions when files are added, renamed or moved.
31
# 4. when merged-in versions are observed, walk down through them
32
# to discover everything, then commit bottom-up
34
# 5. track ancestry as things are merged in, and commit that in each
37
# Perhaps it's best to first walk the whole graph and make a plan for
38
# what should be imported in what order? Need a kind of topological
39
# sort of all revisions. (Or do we, can we just before doing a revision
40
# see that all its parents have either been converted or abandoned?)
43
# Cannot import a revision until all its parents have been
44
# imported. in other words, we can only import revisions whose
45
# parents have all been imported. the first step must be to
46
# import a revision with no parents, of which there must be at
47
# least one. (So perhaps it's useful to store forward pointers
48
# from a list of parents to their children?)
50
# Another (equivalent?) approach is to build up the ordered
51
# ancestry list for the last revision, and walk through that. We
52
# are going to need that.
54
# We don't want to have to recurse all the way back down the list.
56
# Suppose we keep a queue of the revisions able to be processed at
57
# any point. This starts out with all the revisions having no
60
# This seems like a generally useful algorithm...
62
# The current algorithm is dumb (O(n**2)?) but will do the job, and
63
# takes less than a second on the bzr.dev branch.
65
# This currently does a kind of lazy conversion of file texts, where a
66
# new text is written in every version. That's unnecessary but for
67
# the moment saves us having to worry about when files need new
80
import hotshot, hotshot.stats
85
from bzrlib.branch import Branch, find_branch
86
from bzrlib.revfile import Revfile
87
from bzrlib.weave import Weave
88
from bzrlib.weavefile import read_weave, write_weave
89
from bzrlib.progress import ProgressBar
90
from bzrlib.atomicfile import AtomicFile
91
from bzrlib.xml4 import serializer_v4
92
from bzrlib.xml5 import serializer_v5
93
from bzrlib.trace import mutter, note, warning, enable_default_logging
94
from bzrlib.osutils import sha_strings, sha_string
98
class Convert(object):
100
self.converted_revs = set()
101
self.absent_revisions = set()
104
self.inventories = {}
112
enable_default_logging()
113
self.pb = ProgressBar()
114
self.inv_weave = Weave('__inventory')
115
self.anc_weave = Weave('__ancestry')
119
# holds in-memory weaves for all files
120
self.text_weaves = {}
122
b = self.branch = Branch('.', relax_version_check=True)
125
rev_history = b.revision_history()
129
# to_read is a stack holding the revisions we still need to process;
130
# appending to it adds new highest-priority revisions
132
self.known_revisions = set(rev_history)
133
self.to_read = [rev_history[-1]]
135
rev_id = self.to_read.pop()
136
if (rev_id not in self.revisions
137
and rev_id not in self.absent_revisions):
138
self._load_one_rev(rev_id)
140
to_import = self._make_order()
141
for i, rev_id in enumerate(to_import):
142
self.pb.update('converting revision', i, len(to_import))
143
self._convert_one_rev(rev_id)
145
print 'upgraded to weaves:'
146
print ' %6d revisions and inventories' % len(self.revisions)
147
print ' %6d absent revisions removed' % len(self.absent_revisions)
148
print ' %6d texts' % self.text_count
149
self._write_all_weaves()
150
self._write_all_revs()
153
def _write_all_weaves(self):
155
write_a_weave(self.inv_weave, 'weaves/inventory.weave')
157
for file_id, file_weave in self.text_weaves.items():
158
self.pb.update('writing weave', i, len(self.text_weaves))
159
write_a_weave(file_weave, 'weaves/%s.weave' % file_id)
163
## write_atomic_weave(self.anc_weave, 'weaves/ancestry.weave')
166
def _write_all_revs(self):
167
"""Write all revisions out in new form."""
169
for i, rev_id in enumerate(self.converted_revs):
170
self.pb.update('write revision', i, len(self.converted_revs))
171
f = file('new-revisions/%s' % rev_id, 'wb')
173
serializer_v5.write_revision(self.revisions[rev_id], f)
180
def _load_one_rev(self, rev_id):
181
"""Load a revision object into memory.
183
Any parents not either loaded or abandoned get queued to be
185
self.pb.update('loading revision',
187
len(self.known_revisions))
188
if rev_id not in self.branch.revision_store:
190
note('revision {%s} not present in branch; '
191
'will not be converted',
193
self.absent_revisions.add(rev_id)
195
rev_xml = self.branch.revision_store[rev_id].read()
196
rev = serializer_v4.read_revision_from_string(rev_xml)
197
for parent_id in rev.parent_ids:
198
self.known_revisions.add(parent_id)
199
self.to_read.append(parent_id)
200
self.revisions[rev_id] = rev
201
old_inv_xml = self.branch.inventory_store[rev_id].read()
202
inv = serializer_v4.read_inventory_from_string(old_inv_xml)
203
assert rev.inventory_sha1 == sha_string(old_inv_xml)
204
self.inventories[rev_id] = inv
207
def _convert_one_rev(self, rev_id):
208
"""Convert revision and all referenced objects to new format."""
209
rev = self.revisions[rev_id]
210
inv = self.inventories[rev_id]
211
self._convert_revision_contents(rev, inv)
212
# the XML is now updated with text versions
213
new_inv_xml = serializer_v5.write_inventory_to_string(inv)
214
inv_parents = [x for x in self.revisions[rev_id].parent_ids
215
if x not in self.absent_revisions]
216
new_inv_sha1 = sha_string(new_inv_xml)
217
self.inv_weave.add(rev_id, inv_parents,
218
new_inv_xml.splitlines(True),
220
# TODO: Upgrade revision XML and write that out
221
rev.inventory_sha1 = new_inv_sha1
222
self.converted_revs.add(rev_id)
225
def _convert_revision_contents(self, rev, inv):
226
"""Convert all the files within a revision.
228
Also upgrade the inventory to refer to the text revision ids."""
229
rev_id = rev.revision_id
230
mutter('converting texts of revision {%s}',
234
if ie.kind != 'file':
236
self._convert_file_version(rev, ie)
237
# TODO: Check and convert name versions
240
def _convert_file_version(self, rev, ie):
241
"""Convert one version of one file.
243
The file needs to be added into the weave if it is a merge
244
of >=2 parents or if it's changed from its parent.
247
rev_id = rev.revision_id
248
w = self.text_weaves.get(file_id)
251
self.text_weaves[file_id] = w
252
file_lines = self.branch.text_store[ie.text_id].readlines()
253
assert sha_strings(file_lines) == ie.text_sha1
254
assert sum(map(len, file_lines)) == ie.text_size
257
for parent_id in rev.parent_ids:
258
if parent_id in self.absent_revisions:
260
assert parent_id in self.converted_revs
261
parent_inv = self.inventories[parent_id]
262
if parent_inv.has_id(file_id):
263
parent_ie = parent_inv[file_id]
264
old_text_version = parent_ie.text_version
265
assert old_text_version in self.converted_revs
266
if old_text_version not in file_parents:
267
file_parents.append(old_text_version)
268
if parent_ie.text_sha1 != ie.text_sha1:
270
if len(file_parents) != 1 or text_changed:
271
w.add(rev_id, file_parents, file_lines, ie.text_sha1)
272
ie.name_version = ie.text_version = rev_id
274
##mutter('import text {%s} of {%s}',
275
## ie.text_id, file_id)
277
##mutter('text of {%s} unchanged from parent', file_id)
278
ie.text_version = file_parents[0]
279
ie.name_version = file_parents[0]
284
def _make_order(self):
285
"""Return a suitable order for importing revisions.
287
The order must be such that an revision is imported after all
288
its (present) parents.
290
todo = set(self.revisions.keys())
291
done = self.absent_revisions.copy()
294
# scan through looking for a revision whose parents
296
for rev_id in sorted(list(todo)):
297
rev = self.revisions[rev_id]
298
parent_ids = set(rev.parent_ids)
299
if parent_ids.issubset(done):
300
# can take this one now
307
def write_a_weave(weave, filename):
308
inv_wf = file(filename, 'wb')
310
write_weave(weave, inv_wf)
317
def profile_convert():
318
prof_f = tempfile.NamedTemporaryFile()
320
prof = hotshot.Profile(prof_f.name)
322
prof.runcall(Convert)
325
stats = hotshot.stats.load(prof_f.name)
327
stats.sort_stats('time')
328
# XXX: Might like to write to stderr or the trace file instead but
329
# print_stats seems hardcoded to stdout
330
stats.print_stats(100)
333
enable_default_logging()
335
if '-p' in sys.argv[1:]:
added
removed
tools/history2weaves.py
