1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from __future__ import absolute_import
19
"""Reconcilers are able to fix some potential data errors in a branch."""
34
revision as _mod_revision,
37
from bzrlib.trace import mutter
38
from bzrlib.tsort import topo_sort
39
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
from bzrlib.i18n import gettext
43
def reconcile(dir, canonicalize_chks=False):
44
"""Reconcile the data in dir.
46
Currently this is limited to a inventory 'reweave'.
48
This is a convenience method, for using a Reconciler object.
50
Directly using Reconciler is recommended for library users that
51
desire fine grained control or analysis of the found issues.
53
:param canonicalize_chks: Make sure CHKs are in canonical form.
55
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
56
reconciler.reconcile()
59
class Reconciler(object):
60
"""Reconcilers are used to reconcile existing data."""
62
def __init__(self, dir, other=None, canonicalize_chks=False):
63
"""Create a Reconciler."""
65
self.canonicalize_chks = canonicalize_chks
68
"""Perform reconciliation.
70
After reconciliation the following attributes document found issues:
72
* `inconsistent_parents`: The number of revisions in the repository
73
whose ancestry was being reported incorrectly.
74
* `garbage_inventories`: The number of inventory objects without
75
revisions that were garbage collected.
76
* `fixed_branch_history`: None if there was no branch, False if the
77
branch history was correct, True if the branch history needed to be
80
self.pb = ui.ui_factory.nested_progress_bar()
87
"""Helper function for performing reconciliation."""
88
self._reconcile_branch()
89
self._reconcile_repository()
91
def _reconcile_branch(self):
93
self.branch = self.bzrdir.open_branch()
94
except errors.NotBranchError:
95
# Nothing to check here
96
self.fixed_branch_history = None
98
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
99
branch_reconciler = self.branch.reconcile(thorough=True)
100
self.fixed_branch_history = branch_reconciler.fixed_history
102
def _reconcile_repository(self):
103
self.repo = self.bzrdir.find_repository()
104
ui.ui_factory.note(gettext('Reconciling repository %s') %
106
self.pb.update(gettext("Reconciling repository"), 0, 1)
107
if self.canonicalize_chks:
109
self.repo.reconcile_canonicalize_chks
110
except AttributeError:
111
raise errors.BzrError(
112
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
113
repo_reconciler = self.repo.reconcile_canonicalize_chks()
115
repo_reconciler = self.repo.reconcile(thorough=True)
116
self.inconsistent_parents = repo_reconciler.inconsistent_parents
117
self.garbage_inventories = repo_reconciler.garbage_inventories
118
if repo_reconciler.aborted:
119
ui.ui_factory.note(gettext(
120
'Reconcile aborted: revision index has inconsistent parents.'))
121
ui.ui_factory.note(gettext(
122
'Run "bzr check" for more details.'))
124
ui.ui_factory.note(gettext('Reconciliation complete.'))
127
class BranchReconciler(object):
128
"""Reconciler that works on a branch."""
130
def __init__(self, a_branch, thorough=False):
131
self.fixed_history = None
132
self.thorough = thorough
133
self.branch = a_branch
136
operation = cleanup.OperationWithCleanups(self._reconcile)
137
self.add_cleanup = operation.add_cleanup
138
operation.run_simple()
140
def _reconcile(self):
141
self.branch.lock_write()
142
self.add_cleanup(self.branch.unlock)
143
self.pb = ui.ui_factory.nested_progress_bar()
144
self.add_cleanup(self.pb.finished)
145
self._reconcile_steps()
147
def _reconcile_steps(self):
148
self._reconcile_revision_history()
150
def _reconcile_revision_history(self):
151
last_revno, last_revision_id = self.branch.last_revision_info()
153
graph = self.branch.repository.get_graph()
155
for revid in graph.iter_lefthand_ancestry(
156
last_revision_id, (_mod_revision.NULL_REVISION,)):
157
real_history.append(revid)
158
except errors.RevisionNotPresent:
159
pass # Hit a ghost left hand parent
160
real_history.reverse()
161
if last_revno != len(real_history):
162
self.fixed_history = True
163
# Technically for Branch5 formats, it is more efficient to use
164
# set_revision_history, as this will regenerate it again.
165
# Not really worth a whole BranchReconciler class just for this,
167
ui.ui_factory.note(gettext('Fixing last revision info {0} '\
169
last_revno, len(real_history)))
170
self.branch.set_last_revision_info(len(real_history),
173
self.fixed_history = False
174
ui.ui_factory.note(gettext('revision_history ok.'))
177
class RepoReconciler(object):
178
"""Reconciler that reconciles a repository.
180
The goal of repository reconciliation is to make any derived data
181
consistent with the core data committed by a user. This can involve
182
reindexing, or removing unreferenced data if that can interfere with
183
queries in a given repository.
185
Currently this consists of an inventory reweave with revision cross-checks.
188
def __init__(self, repo, other=None, thorough=False):
189
"""Construct a RepoReconciler.
191
:param thorough: perform a thorough check which may take longer but
192
will correct non-data loss issues such as incorrect
195
self.garbage_inventories = 0
196
self.inconsistent_parents = 0
199
self.thorough = thorough
202
"""Perform reconciliation.
204
After reconciliation the following attributes document found issues:
206
* `inconsistent_parents`: The number of revisions in the repository
207
whose ancestry was being reported incorrectly.
208
* `garbage_inventories`: The number of inventory objects without
209
revisions that were garbage collected.
211
operation = cleanup.OperationWithCleanups(self._reconcile)
212
self.add_cleanup = operation.add_cleanup
213
operation.run_simple()
215
def _reconcile(self):
216
self.repo.lock_write()
217
self.add_cleanup(self.repo.unlock)
218
self.pb = ui.ui_factory.nested_progress_bar()
219
self.add_cleanup(self.pb.finished)
220
self._reconcile_steps()
222
def _reconcile_steps(self):
223
"""Perform the steps to reconcile this repository."""
224
self._reweave_inventory()
226
def _reweave_inventory(self):
227
"""Regenerate the inventory weave for the repository from scratch.
229
This is a smart function: it will only do the reweave if doing it
230
will correct data issues. The self.thorough flag controls whether
231
only data-loss causing issues (!self.thorough) or all issues
232
(self.thorough) are treated as requiring the reweave.
234
transaction = self.repo.get_transaction()
235
self.pb.update(gettext('Reading inventory data'))
236
self.inventory = self.repo.inventories
237
self.revisions = self.repo.revisions
238
# the total set of revisions to process
239
self.pending = set([key[-1] for key in self.revisions.keys()])
241
# mapping from revision_id to parents
243
# errors that we detect
244
self.inconsistent_parents = 0
245
# we need the revision id of each revision and its available parents list
246
self._setup_steps(len(self.pending))
247
for rev_id in self.pending:
248
# put a revision into the graph.
249
self._graph_revision(rev_id)
250
self._check_garbage_inventories()
251
# if there are no inconsistent_parents and
252
# (no garbage inventories or we are not doing a thorough check)
253
if (not self.inconsistent_parents and
254
(not self.garbage_inventories or not self.thorough)):
255
ui.ui_factory.note(gettext('Inventory ok.'))
257
self.pb.update(gettext('Backing up inventory'), 0, 0)
258
self.repo._backup_inventory()
259
ui.ui_factory.note(gettext('Backup inventory created.'))
260
new_inventories = self.repo._temp_inventories()
262
# we have topological order of revisions and non ghost parents ready.
263
self._setup_steps(len(self._rev_graph))
264
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
265
stream = self._change_inv_parents(
266
self.inventory.get_record_stream(revision_keys, 'unordered', True),
267
self._new_inv_parents,
269
new_inventories.insert_record_stream(stream)
270
# if this worked, the set of new_inventories.keys should equal
272
if not (set(new_inventories.keys()) ==
273
set([(revid,) for revid in self.pending])):
274
raise AssertionError()
275
self.pb.update(gettext('Writing weave'))
276
self.repo._activate_new_inventory()
277
self.inventory = None
278
ui.ui_factory.note(gettext('Inventory regenerated.'))
280
def _new_inv_parents(self, revision_key):
281
"""Lookup ghost-filtered parents for revision_key."""
282
# Use the filtered ghostless parents list:
283
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
285
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
286
"""Adapt a record stream to reconcile the parents."""
287
for record in stream:
288
wanted_parents = get_parents(record.key)
289
if wanted_parents and wanted_parents[0] not in all_revision_keys:
290
# The check for the left most parent only handles knit
291
# compressors, but this code only applies to knit and weave
292
# repositories anyway.
293
bytes = record.get_bytes_as('fulltext')
294
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
296
adapted_record = AdapterFactory(record.key, wanted_parents, record)
298
self._reweave_step('adding inventories')
300
def _setup_steps(self, new_total):
301
"""Setup the markers we need to control the progress bar."""
302
self.total = new_total
305
def _graph_revision(self, rev_id):
306
"""Load a revision into the revision graph."""
307
# pick a random revision
308
# analyse revision id rev_id and put it in the stack.
309
self._reweave_step('loading revisions')
310
rev = self.repo.get_revision_reconcile(rev_id)
312
for parent in rev.parent_ids:
313
if self._parent_is_available(parent):
314
parents.append(parent)
316
mutter('found ghost %s', parent)
317
self._rev_graph[rev_id] = parents
319
def _check_garbage_inventories(self):
320
"""Check for garbage inventories which we cannot trust
322
We cant trust them because their pre-requisite file data may not
323
be present - all we know is that their revision was not installed.
325
if not self.thorough:
327
inventories = set(self.inventory.keys())
328
revisions = set(self.revisions.keys())
329
garbage = inventories.difference(revisions)
330
self.garbage_inventories = len(garbage)
331
for revision_key in garbage:
332
mutter('Garbage inventory {%s} found.', revision_key[-1])
334
def _parent_is_available(self, parent):
335
"""True if parent is a fully available revision
337
A fully available revision has a inventory and a revision object in the
340
if parent in self._rev_graph:
342
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
343
return (inv_present and self.repo.has_revision(parent))
345
def _reweave_step(self, message):
346
"""Mark a single step of regeneration complete."""
347
self.pb.update(message, self.count, self.total)
351
class KnitReconciler(RepoReconciler):
352
"""Reconciler that reconciles a knit format repository.
354
This will detect garbage inventories and remove them in thorough mode.
357
def _reconcile_steps(self):
358
"""Perform the steps to reconcile this repository."""
362
except errors.BzrCheckError:
365
# knits never suffer this
367
self._fix_text_parents()
369
def _load_indexes(self):
370
"""Load indexes for the reconciliation."""
371
self.transaction = self.repo.get_transaction()
372
self.pb.update(gettext('Reading indexes'), 0, 2)
373
self.inventory = self.repo.inventories
374
self.pb.update(gettext('Reading indexes'), 1, 2)
375
self.repo._check_for_inconsistent_revision_parents()
376
self.revisions = self.repo.revisions
377
self.pb.update(gettext('Reading indexes'), 2, 2)
379
def _gc_inventory(self):
380
"""Remove inventories that are not referenced from the revision store."""
381
self.pb.update(gettext('Checking unused inventories'), 0, 1)
382
self._check_garbage_inventories()
383
self.pb.update(gettext('Checking unused inventories'), 1, 3)
384
if not self.garbage_inventories:
385
ui.ui_factory.note(gettext('Inventory ok.'))
387
self.pb.update(gettext('Backing up inventory'), 0, 0)
388
self.repo._backup_inventory()
389
ui.ui_factory.note(gettext('Backup Inventory created'))
390
# asking for '' should never return a non-empty weave
391
new_inventories = self.repo._temp_inventories()
392
# we have topological order of revisions and non ghost parents ready.
393
graph = self.revisions.get_parent_map(self.revisions.keys())
394
revision_keys = topo_sort(graph)
395
revision_ids = [key[-1] for key in revision_keys]
396
self._setup_steps(len(revision_keys))
397
stream = self._change_inv_parents(
398
self.inventory.get_record_stream(revision_keys, 'unordered', True),
401
new_inventories.insert_record_stream(stream)
402
# if this worked, the set of new_inventory_vf.names should equal
403
# the revisionds list
404
if not(set(new_inventories.keys()) == set(revision_keys)):
405
raise AssertionError()
406
self.pb.update(gettext('Writing weave'))
407
self.repo._activate_new_inventory()
408
self.inventory = None
409
ui.ui_factory.note(gettext('Inventory regenerated.'))
411
def _fix_text_parents(self):
412
"""Fix bad versionedfile parent entries.
414
It is possible for the parents entry in a versionedfile entry to be
415
inconsistent with the values in the revision and inventory.
417
This method finds entries with such inconsistencies, corrects their
418
parent lists, and replaces the versionedfile with a corrected version.
420
transaction = self.repo.get_transaction()
421
versions = [key[-1] for key in self.revisions.keys()]
422
mutter('Prepopulating revision text cache with %d revisions',
424
vf_checker = self.repo._get_versioned_file_checker()
425
bad_parents, unused_versions = vf_checker.check_file_version_parents(
426
self.repo.texts, self.pb)
427
text_index = vf_checker.text_index
428
per_id_bad_parents = {}
429
for key in unused_versions:
430
# Ensure that every file with unused versions gets rewritten.
431
# NB: This is really not needed, reconcile != pack.
432
per_id_bad_parents[key[0]] = {}
433
# Generate per-knit/weave data.
434
for key, details in bad_parents.iteritems():
437
knit_parents = tuple([parent[-1] for parent in details[0]])
438
correct_parents = tuple([parent[-1] for parent in details[1]])
439
file_details = per_id_bad_parents.setdefault(file_id, {})
440
file_details[rev_id] = (knit_parents, correct_parents)
441
file_id_versions = {}
442
for text_key in text_index:
443
versions_list = file_id_versions.setdefault(text_key[0], [])
444
versions_list.append(text_key[1])
445
# Do the reconcile of individual weaves.
446
for num, file_id in enumerate(per_id_bad_parents):
447
self.pb.update(gettext('Fixing text parents'), num,
448
len(per_id_bad_parents))
449
versions_with_bad_parents = per_id_bad_parents[file_id]
450
id_unused_versions = set(key[-1] for key in unused_versions
451
if key[0] == file_id)
452
if file_id in file_id_versions:
453
file_versions = file_id_versions[file_id]
455
# This id was present in the disk store but is not referenced
456
# by any revision at all.
458
self._fix_text_parent(file_id, versions_with_bad_parents,
459
id_unused_versions, file_versions)
461
def _fix_text_parent(self, file_id, versions_with_bad_parents,
462
unused_versions, all_versions):
463
"""Fix bad versionedfile entries in a single versioned file."""
464
mutter('fixing text parent: %r (%d versions)', file_id,
465
len(versions_with_bad_parents))
466
mutter('(%d are unused)', len(unused_versions))
467
new_file_id = 'temp:%s' % file_id
470
for version in all_versions:
471
if version in unused_versions:
473
elif version in versions_with_bad_parents:
474
parents = versions_with_bad_parents[version][1]
476
pmap = self.repo.texts.get_parent_map([(file_id, version)])
477
parents = [key[-1] for key in pmap[(file_id, version)]]
478
new_parents[(new_file_id, version)] = [
479
(new_file_id, parent) for parent in parents]
480
needed_keys.add((file_id, version))
481
def fix_parents(stream):
482
for record in stream:
483
bytes = record.get_bytes_as('fulltext')
484
new_key = (new_file_id, record.key[-1])
485
parents = new_parents[new_key]
486
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
487
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
488
self.repo._remove_file_id(new_file_id)
489
self.repo.texts.insert_record_stream(fix_parents(stream))
490
self.repo._remove_file_id(file_id)
492
self.repo._move_file_id(new_file_id, file_id)
495
class PackReconciler(RepoReconciler):
496
"""Reconciler that reconciles a pack based repository.
498
Garbage inventories do not affect ancestry queries, and removal is
499
considerably more expensive as there is no separate versioned file for
500
them, so they are not cleaned. In short it is currently a no-op.
502
In future this may be a good place to hook in annotation cache checking,
503
index recreation etc.
506
# XXX: The index corruption that _fix_text_parents performs is needed for
507
# packs, but not yet implemented. The basic approach is to:
508
# - lock the names list
509
# - perform a customised pack() that regenerates data as needed
510
# - unlock the names list
511
# https://bugs.launchpad.net/bzr/+bug/154173
513
def __init__(self, repo, other=None, thorough=False,
514
canonicalize_chks=False):
515
super(PackReconciler, self).__init__(repo, other=other,
517
self.canonicalize_chks = canonicalize_chks
519
def _reconcile_steps(self):
520
"""Perform the steps to reconcile this repository."""
521
if not self.thorough:
523
collection = self.repo._pack_collection
524
collection.ensure_loaded()
525
collection.lock_names()
526
self.add_cleanup(collection._unlock_names)
527
packs = collection.all_packs()
528
all_revisions = self.repo.all_revision_ids()
529
total_inventories = len(list(
530
collection.inventory_index.combined_index.iter_all_entries()))
531
if len(all_revisions):
532
if self.canonicalize_chks:
533
reconcile_meth = self.repo._canonicalize_chks_pack
535
reconcile_meth = self.repo._reconcile_pack
536
new_pack = reconcile_meth(collection, packs, ".reconcile",
537
all_revisions, self.pb)
538
if new_pack is not None:
539
self._discard_and_save(packs)
541
# only make a new pack when there is data to copy.
542
self._discard_and_save(packs)
543
self.garbage_inventories = total_inventories - len(list(
544
collection.inventory_index.combined_index.iter_all_entries()))
546
def _discard_and_save(self, packs):
547
"""Discard some packs from the repository.
549
This removes them from the memory index, saves the in-memory index
550
which makes the newly reconciled pack visible and hides the packs to be
551
discarded, and finally renames the packs being discarded into the
552
obsolete packs directory.
554
:param packs: The packs to discard.
557
self.repo._pack_collection._remove_pack_from_memory(pack)
558
self.repo._pack_collection._save_pack_names()
559
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
