1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
34
from bzrlib.trace import mutter, note
35
from bzrlib.tsort import TopoSorter
38
def reconcile(dir, other=None):
39
"""Reconcile the data in dir.
41
Currently this is limited to a inventory 'reweave'.
43
This is a convenience method, for using a Reconciler object.
45
Directly using Reconciler is recommended for library users that
46
desire fine grained control or analysis of the found issues.
48
:param other: another bzrdir to reconcile against.
50
reconciler = Reconciler(dir, other=other)
51
reconciler.reconcile()
54
class Reconciler(object):
55
"""Reconcilers are used to reconcile existing data."""
57
def __init__(self, dir, other=None):
58
"""Create a Reconciler."""
62
"""Perform reconciliation.
64
After reconciliation the following attributes document found issues:
65
inconsistent_parents: The number of revisions in the repository whose
66
ancestry was being reported incorrectly.
67
garbage_inventories: The number of inventory objects without revisions
68
that were garbage collected.
70
self.pb = ui.ui_factory.nested_progress_bar()
77
"""Helper function for performing reconciliation."""
78
self.repo = self.bzrdir.find_repository()
79
self.pb.note('Reconciling repository %s',
80
self.repo.bzrdir.root_transport.base)
81
repo_reconciler = self.repo.reconcile(thorough=True)
82
self.inconsistent_parents = repo_reconciler.inconsistent_parents
83
self.garbage_inventories = repo_reconciler.garbage_inventories
84
if repo_reconciler.aborted:
86
'Reconcile aborted: revision index has inconsistent parents.')
88
'Run "bzr check" for more details.')
90
self.pb.note('Reconciliation complete.')
93
class RepoReconciler(object):
94
"""Reconciler that reconciles a repository.
96
The goal of repository reconciliation is to make any derived data
97
consistent with the core data committed by a user. This can involve
98
reindexing, or removing unreferenced data if that can interfere with
99
queries in a given repository.
101
Currently this consists of an inventory reweave with revision cross-checks.
104
def __init__(self, repo, other=None, thorough=False):
105
"""Construct a RepoReconciler.
107
:param thorough: perform a thorough check which may take longer but
108
will correct non-data loss issues such as incorrect
111
self.garbage_inventories = 0
112
self.inconsistent_parents = 0
115
self.thorough = thorough
118
"""Perform reconciliation.
120
After reconciliation the following attributes document found issues:
121
inconsistent_parents: The number of revisions in the repository whose
122
ancestry was being reported incorrectly.
123
garbage_inventories: The number of inventory objects without revisions
124
that were garbage collected.
126
self.repo.lock_write()
128
self.pb = ui.ui_factory.nested_progress_bar()
130
self._reconcile_steps()
136
def _reconcile_steps(self):
137
"""Perform the steps to reconcile this repository."""
138
self._reweave_inventory()
140
def _reweave_inventory(self):
141
"""Regenerate the inventory weave for the repository from scratch.
143
This is a smart function: it will only do the reweave if doing it
144
will correct data issues. The self.thorough flag controls whether
145
only data-loss causing issues (!self.thorough) or all issues
146
(self.thorough) are treated as requiring the reweave.
148
# local because needing to know about WeaveFile is a wart we want to hide
149
from bzrlib.weave import WeaveFile, Weave
150
transaction = self.repo.get_transaction()
151
self.pb.update('Reading inventory data.')
152
self.inventory = self.repo.get_inventory_weave()
153
# the total set of revisions to process
154
self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)])
156
# mapping from revision_id to parents
158
# errors that we detect
159
self.inconsistent_parents = 0
160
# we need the revision id of each revision and its available parents list
161
self._setup_steps(len(self.pending))
162
for rev_id in self.pending:
163
# put a revision into the graph.
164
self._graph_revision(rev_id)
165
self._check_garbage_inventories()
166
# if there are no inconsistent_parents and
167
# (no garbage inventories or we are not doing a thorough check)
168
if (not self.inconsistent_parents and
169
(not self.garbage_inventories or not self.thorough)):
170
self.pb.note('Inventory ok.')
172
self.pb.update('Backing up inventory...', 0, 0)
173
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
174
self.pb.note('Backup Inventory created.')
175
# asking for '' should never return a non-empty weave
176
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
177
self.repo.get_transaction())
179
# we have topological order of revisions and non ghost parents ready.
180
self._setup_steps(len(self._rev_graph))
181
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
182
parents = self._rev_graph[rev_id]
183
# double check this really is in topological order.
184
unavailable = [p for p in parents if p not in new_inventory_vf]
185
assert len(unavailable) == 0
186
# this entry has all the non ghost parents in the inventory
188
self._reweave_step('adding inventories')
189
if isinstance(new_inventory_vf, WeaveFile):
190
# It's really a WeaveFile, but we call straight into the
191
# Weave's add method to disable the auto-write-out behaviour.
192
# This is done to avoid a revision_count * time-to-write additional overhead on
194
new_inventory_vf._check_write_ok()
195
Weave._add_lines(new_inventory_vf, rev_id, parents,
196
self.inventory.get_lines(rev_id), None, None, None, False, True)
198
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
200
if isinstance(new_inventory_vf, WeaveFile):
201
new_inventory_vf._save()
202
# if this worked, the set of new_inventory_vf.names should equal
204
assert set(new_inventory_vf.versions()) == self.pending
205
self.pb.update('Writing weave')
206
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
207
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
208
self.inventory = None
209
self.pb.note('Inventory regenerated.')
211
def _setup_steps(self, new_total):
212
"""Setup the markers we need to control the progress bar."""
213
self.total = new_total
216
def _graph_revision(self, rev_id):
217
"""Load a revision into the revision graph."""
218
# pick a random revision
219
# analyse revision id rev_id and put it in the stack.
220
self._reweave_step('loading revisions')
221
rev = self.repo.get_revision_reconcile(rev_id)
222
assert rev.revision_id == rev_id
224
for parent in rev.parent_ids:
225
if self._parent_is_available(parent):
226
parents.append(parent)
228
mutter('found ghost %s', parent)
229
self._rev_graph[rev_id] = parents
230
if self._parents_are_inconsistent(rev_id, parents):
231
self.inconsistent_parents += 1
232
mutter('Inconsistent inventory parents: id {%s} '
233
'inventory claims %r, '
234
'available parents are %r, '
235
'unavailable parents are %r',
237
set(self.inventory.get_parents(rev_id)),
239
set(rev.parent_ids).difference(set(parents)))
241
def _parents_are_inconsistent(self, rev_id, parents):
242
"""Return True if the parents list of rev_id does not match the weave.
244
This detects inconsistencies based on the self.thorough value:
245
if thorough is on, the first parent value is checked as well as ghost
247
Otherwise only the ghost differences are evaluated.
249
weave_parents = self.inventory.get_parents(rev_id)
250
weave_missing_old_ghosts = set(weave_parents) != set(parents)
251
first_parent_is_wrong = (
252
len(weave_parents) and len(parents) and
253
parents[0] != weave_parents[0])
255
return weave_missing_old_ghosts or first_parent_is_wrong
257
return weave_missing_old_ghosts
259
def _check_garbage_inventories(self):
260
"""Check for garbage inventories which we cannot trust
262
We cant trust them because their pre-requisite file data may not
263
be present - all we know is that their revision was not installed.
265
if not self.thorough:
267
inventories = set(self.inventory.versions())
268
revisions = set(self._rev_graph.keys())
269
garbage = inventories.difference(revisions)
270
self.garbage_inventories = len(garbage)
271
for revision_id in garbage:
272
mutter('Garbage inventory {%s} found.', revision_id)
274
def _parent_is_available(self, parent):
275
"""True if parent is a fully available revision
277
A fully available revision has a inventory and a revision object in the
280
return (parent in self._rev_graph or
281
(parent in self.inventory and self.repo.has_revision(parent)))
283
def _reweave_step(self, message):
284
"""Mark a single step of regeneration complete."""
285
self.pb.update(message, self.count, self.total)
289
class KnitReconciler(RepoReconciler):
290
"""Reconciler that reconciles a knit format repository.
292
This will detect garbage inventories and remove them in thorough mode.
295
def _reconcile_steps(self):
296
"""Perform the steps to reconcile this repository."""
300
except errors.BzrCheckError:
303
# knits never suffer this
305
self._fix_text_parents()
307
def _load_indexes(self):
308
"""Load indexes for the reconciliation."""
309
self.transaction = self.repo.get_transaction()
310
self.pb.update('Reading indexes.', 0, 2)
311
self.inventory = self.repo.get_inventory_weave()
312
self.pb.update('Reading indexes.', 1, 2)
313
self.repo._check_for_inconsistent_revision_parents()
314
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
315
self.pb.update('Reading indexes.', 2, 2)
317
def _gc_inventory(self):
318
"""Remove inventories that are not referenced from the revision store."""
319
self.pb.update('Checking unused inventories.', 0, 1)
320
self._check_garbage_inventories()
321
self.pb.update('Checking unused inventories.', 1, 3)
322
if not self.garbage_inventories:
323
self.pb.note('Inventory ok.')
325
self.pb.update('Backing up inventory...', 0, 0)
326
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
327
self.pb.note('Backup Inventory created.')
328
# asking for '' should never return a non-empty weave
329
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
332
# we have topological order of revisions and non ghost parents ready.
333
self._setup_steps(len(self.revisions))
334
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
335
parents = self.revisions.get_parents(rev_id)
336
# double check this really is in topological order.
337
unavailable = [p for p in parents if p not in new_inventory_vf]
338
assert len(unavailable) == 0
339
# this entry has all the non ghost parents in the inventory
341
self._reweave_step('adding inventories')
342
# ugly but needed, weaves are just way tooooo slow else.
343
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
345
# if this worked, the set of new_inventory_vf.names should equal
347
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
348
self.pb.update('Writing weave')
349
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
350
self.repo.control_weaves.delete('inventory.new', self.transaction)
351
self.inventory = None
352
self.pb.note('Inventory regenerated.')
354
def _check_garbage_inventories(self):
355
"""Check for garbage inventories which we cannot trust
357
We cant trust them because their pre-requisite file data may not
358
be present - all we know is that their revision was not installed.
360
inventories = set(self.inventory.versions())
361
revisions = set(self.revisions.versions())
362
garbage = inventories.difference(revisions)
363
self.garbage_inventories = len(garbage)
364
for revision_id in garbage:
365
mutter('Garbage inventory {%s} found.', revision_id)
367
def _fix_text_parents(self):
368
"""Fix bad versionedfile parent entries.
370
It is possible for the parents entry in a versionedfile entry to be
371
inconsistent with the values in the revision and inventory.
373
This method finds entries with such inconsistencies, corrects their
374
parent lists, and replaces the versionedfile with a corrected version.
376
transaction = self.repo.get_transaction()
377
revision_versions = repository._RevisionTextVersionCache(self.repo)
378
versions = self.revisions.versions()
379
mutter('Prepopulating revision text cache with %d revisions',
381
revision_versions.prepopulate_revs(versions)
382
used_file_versions = revision_versions.used_file_versions()
383
for num, file_id in enumerate(self.repo.weave_store):
384
self.pb.update('Fixing text parents', num,
385
len(self.repo.weave_store))
386
vf = self.repo.weave_store.get_weave(file_id, transaction)
387
vf_checker = self.repo.get_versioned_file_checker(
388
vf.versions(), revision_versions)
389
versions_with_bad_parents, dangling_file_versions = \
390
vf_checker.check_file_version_parents(vf, file_id)
391
if (len(versions_with_bad_parents) == 0 and
392
len(dangling_file_versions) == 0):
394
full_text_versions = set()
395
unused_versions = set()
396
for dangling_version in dangling_file_versions:
397
version = dangling_version[1]
398
if dangling_version in used_file_versions:
399
# This version *is* used by some revision, even though it
400
# isn't used by its own revision! We make sure any
401
# revision referencing it is stored as a fulltext
402
# This avoids bug 155730: it means that clients looking at
403
# inventories to determine the versions to fetch will not
404
# miss a required version. (So clients can assume that if
405
# they have a complete revision graph, and fetch all file
406
# versions named by those revisions inventories, then they
407
# will not have any missing parents for 'delta' knit
409
# XXX: A better, but more difficult and slower fix would be
410
# to rewrite the inventories referencing this version.
411
full_text_versions.add(version)
413
# This version is totally unreferenced. It should be
415
unused_versions.add(version)
416
self._fix_text_parent(file_id, vf, versions_with_bad_parents,
417
full_text_versions, unused_versions)
419
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents,
420
full_text_versions, unused_versions):
421
"""Fix bad versionedfile entries in a single versioned file."""
422
mutter('fixing text parent: %r (%d versions)', file_id,
423
len(versions_with_bad_parents))
424
mutter('(%d need to be full texts, %d are unused)',
425
len(full_text_versions), len(unused_versions))
426
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
429
for version in vf.versions():
430
if version in versions_with_bad_parents:
431
parents = versions_with_bad_parents[version][1]
433
parents = vf.get_parents(version)
434
new_parents[version] = parents
435
for version in TopoSorter(new_parents.items()).iter_topo_order():
436
if version in unused_versions:
438
lines = vf.get_lines(version)
439
parents = new_parents[version]
440
if parents and (parents[0] in full_text_versions):
441
# Force this record to be a fulltext, not a delta.
442
new_vf._add(version, lines, parents, False,
443
None, None, None, False)
445
new_vf.add_lines(version, parents, lines)
446
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
447
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
450
class PackReconciler(RepoReconciler):
451
"""Reconciler that reconciles a pack based repository.
453
Garbage inventories do not affect ancestry queries, and removal is
454
considerably more expensive as there is no separate versioned file for
455
them, so they are not cleaned. In short it is currently a no-op.
457
In future this may be a good place to hook in annotation cache checking,
458
index recreation etc.
461
# XXX: The index corruption that _fix_text_parents performs is needed for
462
# packs, but not yet implemented. The basic approach is to:
463
# - lock the names list
464
# - perform a customised pack() that regenerates data as needed
465
# - unlock the names list
466
# https://bugs.edge.launchpad.net/bzr/+bug/154173
468
def _reconcile_steps(self):
469
"""Perform the steps to reconcile this repository."""
added
removed
bzrlib/reconcile.py
