1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
20
__all__ = ['reconcile', 'Reconciler', 'RepoReconciler', 'KnitReconciler']
29
from bzrlib import errors
31
from bzrlib.trace import mutter, note
32
from bzrlib.tsort import TopoSorter, topo_sort
35
def reconcile(dir, other=None):
36
"""Reconcile the data in dir.
38
Currently this is limited to a inventory 'reweave'.
40
This is a convenience method, for using a Reconciler object.
42
Directly using Reconciler is recommended for library users that
43
desire fine grained control or analysis of the found issues.
45
:param other: another bzrdir to reconcile against.
47
reconciler = Reconciler(dir, other=other)
48
reconciler.reconcile()
51
class Reconciler(object):
52
"""Reconcilers are used to reconcile existing data."""
54
def __init__(self, dir, other=None):
55
"""Create a Reconciler."""
59
"""Perform reconciliation.
61
After reconciliation the following attributes document found issues:
62
inconsistent_parents: The number of revisions in the repository whose
63
ancestry was being reported incorrectly.
64
garbage_inventories: The number of inventory objects without revisions
65
that were garbage collected.
67
self.pb = ui.ui_factory.nested_progress_bar()
74
"""Helper function for performing reconciliation."""
75
self.repo = self.bzrdir.find_repository()
76
self.pb.note('Reconciling repository %s',
77
self.repo.bzrdir.root_transport.base)
78
repo_reconciler = self.repo.reconcile(thorough=True)
79
self.inconsistent_parents = repo_reconciler.inconsistent_parents
80
self.garbage_inventories = repo_reconciler.garbage_inventories
81
if repo_reconciler.aborted:
83
'Reconcile aborted: revision index has inconsistent parents.')
85
'Run "bzr check" for more details.')
87
self.pb.note('Reconciliation complete.')
90
class RepoReconciler(object):
91
"""Reconciler that reconciles a repository.
93
The goal of repository reconciliation is to make any derived data
94
consistent with the core data committed by a user. This can involve
95
reindexing, or removing unreferenced data if that can interfere with
96
queries in a given repository.
98
Currently this consists of an inventory reweave with revision cross-checks.
101
def __init__(self, repo, other=None, thorough=False):
102
"""Construct a RepoReconciler.
104
:param thorough: perform a thorough check which may take longer but
105
will correct non-data loss issues such as incorrect
108
self.garbage_inventories = 0
109
self.inconsistent_parents = 0
112
self.thorough = thorough
115
"""Perform reconciliation.
117
After reconciliation the following attributes document found issues:
118
inconsistent_parents: The number of revisions in the repository whose
119
ancestry was being reported incorrectly.
120
garbage_inventories: The number of inventory objects without revisions
121
that were garbage collected.
123
self.repo.lock_write()
125
self.pb = ui.ui_factory.nested_progress_bar()
127
self._reconcile_steps()
133
def _reconcile_steps(self):
134
"""Perform the steps to reconcile this repository."""
135
self._reweave_inventory()
137
def _reweave_inventory(self):
138
"""Regenerate the inventory weave for the repository from scratch.
140
This is a smart function: it will only do the reweave if doing it
141
will correct data issues. The self.thorough flag controls whether
142
only data-loss causing issues (!self.thorough) or all issues
143
(self.thorough) are treated as requiring the reweave.
145
# local because needing to know about WeaveFile is a wart we want to hide
146
from bzrlib.weave import WeaveFile, Weave
147
transaction = self.repo.get_transaction()
148
self.pb.update('Reading inventory data.')
149
self.inventory = self.repo.get_inventory_weave()
150
# the total set of revisions to process
151
self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)])
153
# mapping from revision_id to parents
155
# errors that we detect
156
self.inconsistent_parents = 0
157
# we need the revision id of each revision and its available parents list
158
self._setup_steps(len(self.pending))
159
for rev_id in self.pending:
160
# put a revision into the graph.
161
self._graph_revision(rev_id)
162
self._check_garbage_inventories()
163
# if there are no inconsistent_parents and
164
# (no garbage inventories or we are not doing a thorough check)
165
if (not self.inconsistent_parents and
166
(not self.garbage_inventories or not self.thorough)):
167
self.pb.note('Inventory ok.')
169
self.pb.update('Backing up inventory...', 0, 0)
170
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
171
self.pb.note('Backup Inventory created.')
172
# asking for '' should never return a non-empty weave
173
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
174
self.repo.get_transaction())
176
# we have topological order of revisions and non ghost parents ready.
177
self._setup_steps(len(self._rev_graph))
178
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
179
parents = self._rev_graph[rev_id]
180
# double check this really is in topological order.
181
unavailable = [p for p in parents if p not in new_inventory_vf]
182
assert len(unavailable) == 0
183
# this entry has all the non ghost parents in the inventory
185
self._reweave_step('adding inventories')
186
if isinstance(new_inventory_vf, WeaveFile):
187
# It's really a WeaveFile, but we call straight into the
188
# Weave's add method to disable the auto-write-out behaviour.
189
# This is done to avoid a revision_count * time-to-write additional overhead on
191
new_inventory_vf._check_write_ok()
192
Weave._add_lines(new_inventory_vf, rev_id, parents,
193
self.inventory.get_lines(rev_id), None, None, None, False, True)
195
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
197
if isinstance(new_inventory_vf, WeaveFile):
198
new_inventory_vf._save()
199
# if this worked, the set of new_inventory_vf.names should equal
201
assert set(new_inventory_vf.versions()) == self.pending
202
self.pb.update('Writing weave')
203
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
204
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
205
self.inventory = None
206
self.pb.note('Inventory regenerated.')
208
def _setup_steps(self, new_total):
209
"""Setup the markers we need to control the progress bar."""
210
self.total = new_total
213
def _graph_revision(self, rev_id):
214
"""Load a revision into the revision graph."""
215
# pick a random revision
216
# analyse revision id rev_id and put it in the stack.
217
self._reweave_step('loading revisions')
218
rev = self.repo.get_revision_reconcile(rev_id)
219
assert rev.revision_id == rev_id
221
for parent in rev.parent_ids:
222
if self._parent_is_available(parent):
223
parents.append(parent)
225
mutter('found ghost %s', parent)
226
self._rev_graph[rev_id] = parents
227
if self._parents_are_inconsistent(rev_id, parents):
228
self.inconsistent_parents += 1
229
mutter('Inconsistent inventory parents: id {%s} '
230
'inventory claims %r, '
231
'available parents are %r, '
232
'unavailable parents are %r',
234
set(self.inventory.get_parents(rev_id)),
236
set(rev.parent_ids).difference(set(parents)))
238
def _parents_are_inconsistent(self, rev_id, parents):
239
"""Return True if the parents list of rev_id does not match the weave.
241
This detects inconsistencies based on the self.thorough value:
242
if thorough is on, the first parent value is checked as well as ghost
244
Otherwise only the ghost differences are evaluated.
246
weave_parents = self.inventory.get_parents(rev_id)
247
weave_missing_old_ghosts = set(weave_parents) != set(parents)
248
first_parent_is_wrong = (
249
len(weave_parents) and len(parents) and
250
parents[0] != weave_parents[0])
252
return weave_missing_old_ghosts or first_parent_is_wrong
254
return weave_missing_old_ghosts
256
def _check_garbage_inventories(self):
257
"""Check for garbage inventories which we cannot trust
259
We cant trust them because their pre-requisite file data may not
260
be present - all we know is that their revision was not installed.
262
if not self.thorough:
264
inventories = set(self.inventory.versions())
265
revisions = set(self._rev_graph.keys())
266
garbage = inventories.difference(revisions)
267
self.garbage_inventories = len(garbage)
268
for revision_id in garbage:
269
mutter('Garbage inventory {%s} found.', revision_id)
271
def _parent_is_available(self, parent):
272
"""True if parent is a fully available revision
274
A fully available revision has a inventory and a revision object in the
277
return (parent in self._rev_graph or
278
(parent in self.inventory and self.repo.has_revision(parent)))
280
def _reweave_step(self, message):
281
"""Mark a single step of regeneration complete."""
282
self.pb.update(message, self.count, self.total)
286
class KnitReconciler(RepoReconciler):
287
"""Reconciler that reconciles a knit format repository.
289
This will detect garbage inventories and remove them in thorough mode.
292
def _reconcile_steps(self):
293
"""Perform the steps to reconcile this repository."""
297
except errors.BzrCheckError:
300
# knits never suffer this
302
self._fix_text_parents()
304
def _load_indexes(self):
305
"""Load indexes for the reconciliation."""
306
self.transaction = self.repo.get_transaction()
307
self.pb.update('Reading indexes.', 0, 2)
308
self.inventory = self.repo.get_inventory_weave()
309
self.pb.update('Reading indexes.', 1, 2)
310
self.repo._check_for_inconsistent_revision_parents()
311
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
312
self.pb.update('Reading indexes.', 2, 2)
314
def _gc_inventory(self):
315
"""Remove inventories that are not referenced from the revision store."""
316
self.pb.update('Checking unused inventories.', 0, 1)
317
self._check_garbage_inventories()
318
self.pb.update('Checking unused inventories.', 1, 3)
319
if not self.garbage_inventories:
320
self.pb.note('Inventory ok.')
322
self.pb.update('Backing up inventory...', 0, 0)
323
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
324
self.pb.note('Backup Inventory created.')
325
# asking for '' should never return a non-empty weave
326
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
329
# we have topological order of revisions and non ghost parents ready.
330
self._setup_steps(len(self.revisions))
331
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
332
parents = self.revisions.get_parents(rev_id)
333
# double check this really is in topological order.
334
unavailable = [p for p in parents if p not in new_inventory_vf]
335
assert len(unavailable) == 0
336
# this entry has all the non ghost parents in the inventory
338
self._reweave_step('adding inventories')
339
# ugly but needed, weaves are just way tooooo slow else.
340
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
342
# if this worked, the set of new_inventory_vf.names should equal
344
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
345
self.pb.update('Writing weave')
346
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
347
self.repo.control_weaves.delete('inventory.new', self.transaction)
348
self.inventory = None
349
self.pb.note('Inventory regenerated.')
351
def _check_garbage_inventories(self):
352
"""Check for garbage inventories which we cannot trust
354
We cant trust them because their pre-requisite file data may not
355
be present - all we know is that their revision was not installed.
357
inventories = set(self.inventory.versions())
358
revisions = set(self.revisions.versions())
359
garbage = inventories.difference(revisions)
360
self.garbage_inventories = len(garbage)
361
for revision_id in garbage:
362
mutter('Garbage inventory {%s} found.', revision_id)
364
def _fix_text_parents(self):
365
"""Fix bad versionedfile parent entries.
367
It is possible for the parents entry in a versionedfile entry to be
368
inconsistent with the values in the revision and inventory.
370
This method finds entries with such inconsistencies, corrects their
371
parent lists, and replaces the versionedfile with a corrected version.
373
transaction = self.repo.get_transaction()
374
revision_versions = repository._RevisionTextVersionCache(self.repo)
375
versions = self.revisions.versions()
376
revision_versions.prepopulate_revs(versions)
377
for num, file_id in enumerate(self.repo.weave_store):
378
self.pb.update('Fixing text parents', num,
379
len(self.repo.weave_store))
380
vf = self.repo.weave_store.get_weave(file_id, transaction)
381
vf_checker = self.repo.get_versioned_file_checker(
382
versions, revision_versions)
383
versions_with_bad_parents = vf_checker.check_file_version_parents(
385
if len(versions_with_bad_parents) == 0:
387
self._fix_text_parent(file_id, vf, versions_with_bad_parents)
389
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents):
390
"""Fix bad versionedfile entries in a single versioned file."""
391
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
394
for version in vf.versions():
395
if version in versions_with_bad_parents:
396
parents = versions_with_bad_parents[version][1]
398
parents = vf.get_parents(version)
399
new_parents[version] = parents
400
for version in topo_sort(new_parents.items()):
401
new_vf.add_lines(version, new_parents[version],
402
vf.get_lines(version))
403
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
404
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
added
removed
bzrlib/reconcile.py
