1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
19
from __future__ import absolute_import
33
revision as _mod_revision,
36
from .trace import mutter
37
from .tsort import topo_sort
38
from .bzr.versionedfile import AdapterFactory, FulltextContentFactory
39
from .i18n import gettext
42
def reconcile(dir, canonicalize_chks=False):
43
"""Reconcile the data in dir.
45
Currently this is limited to a inventory 'reweave'.
47
This is a convenience method, for using a Reconciler object.
49
Directly using Reconciler is recommended for library users that
50
desire fine grained control or analysis of the found issues.
52
:param canonicalize_chks: Make sure CHKs are in canonical form.
54
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
55
reconciler.reconcile()
58
class Reconciler(object):
59
"""Reconcilers are used to reconcile existing data."""
61
def __init__(self, dir, other=None, canonicalize_chks=False):
62
"""Create a Reconciler."""
64
self.canonicalize_chks = canonicalize_chks
67
"""Perform reconciliation.
69
After reconciliation the following attributes document found issues:
71
* `inconsistent_parents`: The number of revisions in the repository
72
whose ancestry was being reported incorrectly.
73
* `garbage_inventories`: The number of inventory objects without
74
revisions that were garbage collected.
75
* `fixed_branch_history`: None if there was no branch, False if the
76
branch history was correct, True if the branch history needed to be
79
operation = cleanup.OperationWithCleanups(self._reconcile)
80
self.add_cleanup = operation.add_cleanup
81
operation.run_simple()
84
"""Helper function for performing reconciliation."""
85
self.pb = ui.ui_factory.nested_progress_bar()
86
self.add_cleanup(self.pb.finished)
87
self._reconcile_branch()
88
self._reconcile_repository()
90
def _reconcile_branch(self):
92
self.branch = self.controldir.open_branch()
93
except errors.NotBranchError:
94
# Nothing to check here
95
self.fixed_branch_history = None
97
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
98
branch_reconciler = self.branch.reconcile(thorough=True)
99
self.fixed_branch_history = branch_reconciler.fixed_history
101
def _reconcile_repository(self):
102
self.repo = self.controldir.find_repository()
103
ui.ui_factory.note(gettext('Reconciling repository %s') %
105
self.pb.update(gettext("Reconciling repository"), 0, 1)
106
if self.canonicalize_chks:
108
self.repo.reconcile_canonicalize_chks
109
except AttributeError:
110
raise errors.BzrError(
111
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
112
repo_reconciler = self.repo.reconcile_canonicalize_chks()
114
repo_reconciler = self.repo.reconcile(thorough=True)
115
self.inconsistent_parents = repo_reconciler.inconsistent_parents
116
self.garbage_inventories = repo_reconciler.garbage_inventories
117
if repo_reconciler.aborted:
118
ui.ui_factory.note(gettext(
119
'Reconcile aborted: revision index has inconsistent parents.'))
120
ui.ui_factory.note(gettext(
121
'Run "brz check" for more details.'))
123
ui.ui_factory.note(gettext('Reconciliation complete.'))
126
class BranchReconciler(object):
127
"""Reconciler that works on a branch."""
129
def __init__(self, a_branch, thorough=False):
130
self.fixed_history = None
131
self.thorough = thorough
132
self.branch = a_branch
135
operation = cleanup.OperationWithCleanups(self._reconcile)
136
self.add_cleanup = operation.add_cleanup
137
operation.run_simple()
139
def _reconcile(self):
140
self.branch.lock_write()
141
self.add_cleanup(self.branch.unlock)
142
self.pb = ui.ui_factory.nested_progress_bar()
143
self.add_cleanup(self.pb.finished)
144
self._reconcile_steps()
146
def _reconcile_steps(self):
147
self._reconcile_revision_history()
149
def _reconcile_revision_history(self):
150
last_revno, last_revision_id = self.branch.last_revision_info()
152
graph = self.branch.repository.get_graph()
154
for revid in graph.iter_lefthand_ancestry(
155
last_revision_id, (_mod_revision.NULL_REVISION,)):
156
real_history.append(revid)
157
except errors.RevisionNotPresent:
158
pass # Hit a ghost left hand parent
159
real_history.reverse()
160
if last_revno != len(real_history):
161
self.fixed_history = True
162
# Technically for Branch5 formats, it is more efficient to use
163
# set_revision_history, as this will regenerate it again.
164
# Not really worth a whole BranchReconciler class just for this,
166
ui.ui_factory.note(gettext('Fixing last revision info {0} '
168
last_revno, len(real_history)))
169
self.branch.set_last_revision_info(len(real_history),
172
self.fixed_history = False
173
ui.ui_factory.note(gettext('revision_history ok.'))
176
class RepoReconciler(object):
177
"""Reconciler that reconciles a repository.
179
The goal of repository reconciliation is to make any derived data
180
consistent with the core data committed by a user. This can involve
181
reindexing, or removing unreferenced data if that can interfere with
182
queries in a given repository.
184
Currently this consists of an inventory reweave with revision cross-checks.
187
def __init__(self, repo, other=None, thorough=False):
188
"""Construct a RepoReconciler.
190
:param thorough: perform a thorough check which may take longer but
191
will correct non-data loss issues such as incorrect
194
self.garbage_inventories = 0
195
self.inconsistent_parents = 0
198
self.thorough = thorough
201
"""Perform reconciliation.
203
After reconciliation the following attributes document found issues:
205
* `inconsistent_parents`: The number of revisions in the repository
206
whose ancestry was being reported incorrectly.
207
* `garbage_inventories`: The number of inventory objects without
208
revisions that were garbage collected.
210
operation = cleanup.OperationWithCleanups(self._reconcile)
211
self.add_cleanup = operation.add_cleanup
212
operation.run_simple()
214
def _reconcile(self):
215
self.repo.lock_write()
216
self.add_cleanup(self.repo.unlock)
217
self.pb = ui.ui_factory.nested_progress_bar()
218
self.add_cleanup(self.pb.finished)
219
self._reconcile_steps()
221
def _reconcile_steps(self):
222
"""Perform the steps to reconcile this repository."""
223
self._reweave_inventory()
225
def _reweave_inventory(self):
226
"""Regenerate the inventory weave for the repository from scratch.
228
This is a smart function: it will only do the reweave if doing it
229
will correct data issues. The self.thorough flag controls whether
230
only data-loss causing issues (!self.thorough) or all issues
231
(self.thorough) are treated as requiring the reweave.
233
transaction = self.repo.get_transaction()
234
self.pb.update(gettext('Reading inventory data'))
235
self.inventory = self.repo.inventories
236
self.revisions = self.repo.revisions
237
# the total set of revisions to process
238
self.pending = {key[-1] for key in self.revisions.keys()}
240
# mapping from revision_id to parents
242
# errors that we detect
243
self.inconsistent_parents = 0
244
# we need the revision id of each revision and its available parents list
245
self._setup_steps(len(self.pending))
246
for rev_id in self.pending:
247
# put a revision into the graph.
248
self._graph_revision(rev_id)
249
self._check_garbage_inventories()
250
# if there are no inconsistent_parents and
251
# (no garbage inventories or we are not doing a thorough check)
252
if (not self.inconsistent_parents
253
and (not self.garbage_inventories or not self.thorough)):
254
ui.ui_factory.note(gettext('Inventory ok.'))
256
self.pb.update(gettext('Backing up inventory'), 0, 0)
257
self.repo._backup_inventory()
258
ui.ui_factory.note(gettext('Backup inventory created.'))
259
new_inventories = self.repo._temp_inventories()
261
# we have topological order of revisions and non ghost parents ready.
262
self._setup_steps(len(self._rev_graph))
263
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
264
stream = self._change_inv_parents(
265
self.inventory.get_record_stream(revision_keys, 'unordered', True),
266
self._new_inv_parents,
268
new_inventories.insert_record_stream(stream)
269
# if this worked, the set of new_inventories.keys should equal
271
if not (set(new_inventories.keys())
272
== {(revid,) for revid in self.pending}):
273
raise AssertionError()
274
self.pb.update(gettext('Writing weave'))
275
self.repo._activate_new_inventory()
276
self.inventory = None
277
ui.ui_factory.note(gettext('Inventory regenerated.'))
279
def _new_inv_parents(self, revision_key):
280
"""Lookup ghost-filtered parents for revision_key."""
281
# Use the filtered ghostless parents list:
282
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
284
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
285
"""Adapt a record stream to reconcile the parents."""
286
for record in stream:
287
wanted_parents = get_parents(record.key)
288
if wanted_parents and wanted_parents[0] not in all_revision_keys:
289
# The check for the left most parent only handles knit
290
# compressors, but this code only applies to knit and weave
291
# repositories anyway.
292
bytes = record.get_bytes_as('fulltext')
293
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
295
adapted_record = AdapterFactory(
296
record.key, wanted_parents, record)
298
self._reweave_step('adding inventories')
300
def _setup_steps(self, new_total):
301
"""Setup the markers we need to control the progress bar."""
302
self.total = new_total
305
def _graph_revision(self, rev_id):
306
"""Load a revision into the revision graph."""
307
# pick a random revision
308
# analyse revision id rev_id and put it in the stack.
309
self._reweave_step('loading revisions')
310
rev = self.repo.get_revision_reconcile(rev_id)
312
for parent in rev.parent_ids:
313
if self._parent_is_available(parent):
314
parents.append(parent)
316
mutter('found ghost %s', parent)
317
self._rev_graph[rev_id] = parents
319
def _check_garbage_inventories(self):
320
"""Check for garbage inventories which we cannot trust
322
We cant trust them because their pre-requisite file data may not
323
be present - all we know is that their revision was not installed.
325
if not self.thorough:
327
inventories = set(self.inventory.keys())
328
revisions = set(self.revisions.keys())
329
garbage = inventories.difference(revisions)
330
self.garbage_inventories = len(garbage)
331
for revision_key in garbage:
332
mutter('Garbage inventory {%s} found.', revision_key[-1])
334
def _parent_is_available(self, parent):
335
"""True if parent is a fully available revision
337
A fully available revision has a inventory and a revision object in the
340
if parent in self._rev_graph:
342
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
343
return (inv_present and self.repo.has_revision(parent))
345
def _reweave_step(self, message):
346
"""Mark a single step of regeneration complete."""
347
self.pb.update(message, self.count, self.total)
351
class KnitReconciler(RepoReconciler):
352
"""Reconciler that reconciles a knit format repository.
354
This will detect garbage inventories and remove them in thorough mode.
357
def _reconcile_steps(self):
358
"""Perform the steps to reconcile this repository."""
362
except errors.BzrCheckError:
365
# knits never suffer this
367
self._fix_text_parents()
369
def _load_indexes(self):
370
"""Load indexes for the reconciliation."""
371
self.transaction = self.repo.get_transaction()
372
self.pb.update(gettext('Reading indexes'), 0, 2)
373
self.inventory = self.repo.inventories
374
self.pb.update(gettext('Reading indexes'), 1, 2)
375
self.repo._check_for_inconsistent_revision_parents()
376
self.revisions = self.repo.revisions
377
self.pb.update(gettext('Reading indexes'), 2, 2)
379
def _gc_inventory(self):
380
"""Remove inventories that are not referenced from the revision store."""
381
self.pb.update(gettext('Checking unused inventories'), 0, 1)
382
self._check_garbage_inventories()
383
self.pb.update(gettext('Checking unused inventories'), 1, 3)
384
if not self.garbage_inventories:
385
ui.ui_factory.note(gettext('Inventory ok.'))
387
self.pb.update(gettext('Backing up inventory'), 0, 0)
388
self.repo._backup_inventory()
389
ui.ui_factory.note(gettext('Backup Inventory created'))
390
# asking for '' should never return a non-empty weave
391
new_inventories = self.repo._temp_inventories()
392
# we have topological order of revisions and non ghost parents ready.
393
graph = self.revisions.get_parent_map(self.revisions.keys())
394
revision_keys = topo_sort(graph)
395
revision_ids = [key[-1] for key in revision_keys]
396
self._setup_steps(len(revision_keys))
397
stream = self._change_inv_parents(
398
self.inventory.get_record_stream(revision_keys, 'unordered', True),
401
new_inventories.insert_record_stream(stream)
402
# if this worked, the set of new_inventory_vf.names should equal
403
# the revisionds list
404
if not(set(new_inventories.keys()) == set(revision_keys)):
405
raise AssertionError()
406
self.pb.update(gettext('Writing weave'))
407
self.repo._activate_new_inventory()
408
self.inventory = None
409
ui.ui_factory.note(gettext('Inventory regenerated.'))
411
def _fix_text_parents(self):
412
"""Fix bad versionedfile parent entries.
414
It is possible for the parents entry in a versionedfile entry to be
415
inconsistent with the values in the revision and inventory.
417
This method finds entries with such inconsistencies, corrects their
418
parent lists, and replaces the versionedfile with a corrected version.
420
transaction = self.repo.get_transaction()
421
versions = [key[-1] for key in self.revisions.keys()]
422
mutter('Prepopulating revision text cache with %d revisions',
424
vf_checker = self.repo._get_versioned_file_checker()
425
bad_parents, unused_versions = vf_checker.check_file_version_parents(
426
self.repo.texts, self.pb)
427
text_index = vf_checker.text_index
428
per_id_bad_parents = {}
429
for key in unused_versions:
430
# Ensure that every file with unused versions gets rewritten.
431
# NB: This is really not needed, reconcile != pack.
432
per_id_bad_parents[key[0]] = {}
433
# Generate per-knit/weave data.
434
for key, details in bad_parents.items():
437
knit_parents = tuple([parent[-1] for parent in details[0]])
438
correct_parents = tuple([parent[-1] for parent in details[1]])
439
file_details = per_id_bad_parents.setdefault(file_id, {})
440
file_details[rev_id] = (knit_parents, correct_parents)
441
file_id_versions = {}
442
for text_key in text_index:
443
versions_list = file_id_versions.setdefault(text_key[0], [])
444
versions_list.append(text_key[1])
445
# Do the reconcile of individual weaves.
446
for num, file_id in enumerate(per_id_bad_parents):
447
self.pb.update(gettext('Fixing text parents'), num,
448
len(per_id_bad_parents))
449
versions_with_bad_parents = per_id_bad_parents[file_id]
450
id_unused_versions = set(key[-1] for key in unused_versions
451
if key[0] == file_id)
452
if file_id in file_id_versions:
453
file_versions = file_id_versions[file_id]
455
# This id was present in the disk store but is not referenced
456
# by any revision at all.
458
self._fix_text_parent(file_id, versions_with_bad_parents,
459
id_unused_versions, file_versions)
461
def _fix_text_parent(self, file_id, versions_with_bad_parents,
462
unused_versions, all_versions):
463
"""Fix bad versionedfile entries in a single versioned file."""
464
mutter('fixing text parent: %r (%d versions)', file_id,
465
len(versions_with_bad_parents))
466
mutter('(%d are unused)', len(unused_versions))
467
new_file_id = b'temp:%s' % file_id
470
for version in all_versions:
471
if version in unused_versions:
473
elif version in versions_with_bad_parents:
474
parents = versions_with_bad_parents[version][1]
476
pmap = self.repo.texts.get_parent_map([(file_id, version)])
477
parents = [key[-1] for key in pmap[(file_id, version)]]
478
new_parents[(new_file_id, version)] = [
479
(new_file_id, parent) for parent in parents]
480
needed_keys.add((file_id, version))
482
def fix_parents(stream):
483
for record in stream:
484
bytes = record.get_bytes_as('fulltext')
485
new_key = (new_file_id, record.key[-1])
486
parents = new_parents[new_key]
487
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
488
stream = self.repo.texts.get_record_stream(
489
needed_keys, 'topological', True)
490
self.repo._remove_file_id(new_file_id)
491
self.repo.texts.insert_record_stream(fix_parents(stream))
492
self.repo._remove_file_id(file_id)
494
self.repo._move_file_id(new_file_id, file_id)
497
class PackReconciler(RepoReconciler):
498
"""Reconciler that reconciles a pack based repository.
500
Garbage inventories do not affect ancestry queries, and removal is
501
considerably more expensive as there is no separate versioned file for
502
them, so they are not cleaned. In short it is currently a no-op.
504
In future this may be a good place to hook in annotation cache checking,
505
index recreation etc.
508
# XXX: The index corruption that _fix_text_parents performs is needed for
509
# packs, but not yet implemented. The basic approach is to:
510
# - lock the names list
511
# - perform a customised pack() that regenerates data as needed
512
# - unlock the names list
513
# https://bugs.launchpad.net/bzr/+bug/154173
515
def __init__(self, repo, other=None, thorough=False,
516
canonicalize_chks=False):
517
super(PackReconciler, self).__init__(repo, other=other,
519
self.canonicalize_chks = canonicalize_chks
521
def _reconcile_steps(self):
522
"""Perform the steps to reconcile this repository."""
523
if not self.thorough:
525
collection = self.repo._pack_collection
526
collection.ensure_loaded()
527
collection.lock_names()
528
self.add_cleanup(collection._unlock_names)
529
packs = collection.all_packs()
530
all_revisions = self.repo.all_revision_ids()
531
total_inventories = len(list(
532
collection.inventory_index.combined_index.iter_all_entries()))
533
if len(all_revisions):
534
if self.canonicalize_chks:
535
reconcile_meth = self.repo._canonicalize_chks_pack
537
reconcile_meth = self.repo._reconcile_pack
538
new_pack = reconcile_meth(collection, packs, ".reconcile",
539
all_revisions, self.pb)
540
if new_pack is not None:
541
self._discard_and_save(packs)
543
# only make a new pack when there is data to copy.
544
self._discard_and_save(packs)
545
self.garbage_inventories = total_inventories - len(list(
546
collection.inventory_index.combined_index.iter_all_entries()))
548
def _discard_and_save(self, packs):
549
"""Discard some packs from the repository.
551
This removes them from the memory index, saves the in-memory index
552
which makes the newly reconciled pack visible and hides the packs to be
553
discarded, and finally renames the packs being discarded into the
554
obsolete packs directory.
556
:param packs: The packs to discard.
559
self.repo._pack_collection._remove_pack_from_memory(pack)
560
self.repo._pack_collection._save_pack_names()
561
self.repo._pack_collection._obsolete_packs(packs)
added
removed
breezy/reconcile.py
