1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import TopoSorter
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, other=None):
41
"""Reconcile the data in dir.
43
Currently this is limited to a inventory 'reweave'.
45
This is a convenience method, for using a Reconciler object.
47
Directly using Reconciler is recommended for library users that
48
desire fine grained control or analysis of the found issues.
50
:param other: another bzrdir to reconcile against.
52
reconciler = Reconciler(dir, other=other)
53
reconciler.reconcile()
56
class Reconciler(object):
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None):
60
"""Create a Reconciler."""
64
"""Perform reconciliation.
66
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
75
self.pb = ui.ui_factory.nested_progress_bar()
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
self.pb.note('Reconciling branch %s',
95
branch_reconciler = self.branch.reconcile(thorough=True)
96
self.fixed_branch_history = branch_reconciler.fixed_history
98
def _reconcile_repository(self):
99
self.repo = self.bzrdir.find_repository()
100
self.pb.note('Reconciling repository %s',
101
self.repo.bzrdir.root_transport.base)
102
self.pb.update("Reconciling repository", 0, 1)
103
repo_reconciler = self.repo.reconcile(thorough=True)
104
self.inconsistent_parents = repo_reconciler.inconsistent_parents
105
self.garbage_inventories = repo_reconciler.garbage_inventories
106
if repo_reconciler.aborted:
108
'Reconcile aborted: revision index has inconsistent parents.')
110
'Run "bzr check" for more details.')
112
self.pb.note('Reconciliation complete.')
115
class BranchReconciler(object):
116
"""Reconciler that works on a branch."""
118
def __init__(self, a_branch, thorough=False):
119
self.fixed_history = None
120
self.thorough = thorough
121
self.branch = a_branch
124
self.branch.lock_write()
126
self.pb = ui.ui_factory.nested_progress_bar()
128
self._reconcile_steps()
134
def _reconcile_steps(self):
135
self._reconcile_revision_history()
137
def _reconcile_revision_history(self):
138
repo = self.branch.repository
139
last_revno, last_revision_id = self.branch.last_revision_info()
140
real_history = list(repo.iter_reverse_revision_history(
142
real_history.reverse()
143
if last_revno != len(real_history):
144
self.fixed_history = True
145
# Technically for Branch5 formats, it is more efficient to use
146
# set_revision_history, as this will regenerate it again.
147
# Not really worth a whole BranchReconciler class just for this,
149
self.pb.note('Fixing last revision info %s => %s',
150
last_revno, len(real_history))
151
self.branch.set_last_revision_info(len(real_history),
154
self.fixed_history = False
155
self.pb.note('revision_history ok.')
158
class RepoReconciler(object):
159
"""Reconciler that reconciles a repository.
161
The goal of repository reconciliation is to make any derived data
162
consistent with the core data committed by a user. This can involve
163
reindexing, or removing unreferenced data if that can interfere with
164
queries in a given repository.
166
Currently this consists of an inventory reweave with revision cross-checks.
169
def __init__(self, repo, other=None, thorough=False):
170
"""Construct a RepoReconciler.
172
:param thorough: perform a thorough check which may take longer but
173
will correct non-data loss issues such as incorrect
176
self.garbage_inventories = 0
177
self.inconsistent_parents = 0
180
self.thorough = thorough
183
"""Perform reconciliation.
185
After reconciliation the following attributes document found issues:
186
inconsistent_parents: The number of revisions in the repository whose
187
ancestry was being reported incorrectly.
188
garbage_inventories: The number of inventory objects without revisions
189
that were garbage collected.
191
self.repo.lock_write()
193
self.pb = ui.ui_factory.nested_progress_bar()
195
self._reconcile_steps()
201
def _reconcile_steps(self):
202
"""Perform the steps to reconcile this repository."""
203
self._reweave_inventory()
205
def _reweave_inventory(self):
206
"""Regenerate the inventory weave for the repository from scratch.
208
This is a smart function: it will only do the reweave if doing it
209
will correct data issues. The self.thorough flag controls whether
210
only data-loss causing issues (!self.thorough) or all issues
211
(self.thorough) are treated as requiring the reweave.
213
# local because needing to know about WeaveFile is a wart we want to hide
214
from bzrlib.weave import WeaveFile, Weave
215
transaction = self.repo.get_transaction()
216
self.pb.update('Reading inventory data.')
217
self.inventory = self.repo.inventories
218
self.revisions = self.repo.revisions
219
# the total set of revisions to process
220
self.pending = set([key[-1] for key in self.revisions.keys()])
222
# mapping from revision_id to parents
224
# errors that we detect
225
self.inconsistent_parents = 0
226
# we need the revision id of each revision and its available parents list
227
self._setup_steps(len(self.pending))
228
for rev_id in self.pending:
229
# put a revision into the graph.
230
self._graph_revision(rev_id)
231
self._check_garbage_inventories()
232
# if there are no inconsistent_parents and
233
# (no garbage inventories or we are not doing a thorough check)
234
if (not self.inconsistent_parents and
235
(not self.garbage_inventories or not self.thorough)):
236
self.pb.note('Inventory ok.')
238
self.pb.update('Backing up inventory...', 0, 0)
239
self.repo._backup_inventory()
240
self.pb.note('Backup Inventory created.')
241
new_inventories = self.repo._temp_inventories()
243
# we have topological order of revisions and non ghost parents ready.
244
self._setup_steps(len(self._rev_graph))
245
revision_keys = [(rev_id,) for rev_id in
246
TopoSorter(self._rev_graph.items()).iter_topo_order()]
247
stream = self._change_inv_parents(
248
self.inventory.get_record_stream(revision_keys, 'unsorted', True),
249
self._new_inv_parents,
251
new_inventories.insert_record_stream(stream)
252
# if this worked, the set of new_inventories.keys should equal
254
if not (set(new_inventories.keys()) ==
255
set([(revid,) for revid in self.pending])):
256
raise AssertionError()
257
self.pb.update('Writing weave')
258
self.repo._activate_new_inventory()
259
self.inventory = None
260
self.pb.note('Inventory regenerated.')
262
def _new_inv_parents(self, revision_key):
263
"""Lookup ghost-filtered parents for revision_key."""
264
# Use the filtered ghostless parents list:
265
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
267
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
268
"""Adapt a record stream to reconcile the parents."""
269
for record in stream:
270
wanted_parents = get_parents(record.key)
271
if wanted_parents and wanted_parents[0] not in all_revision_keys:
272
# The check for the left most parent only handles knit
273
# compressors, but this code only applies to knit and weave
274
# repositories anyway.
275
bytes = record.get_bytes_as('fulltext')
276
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
278
adapted_record = AdapterFactory(record.key, wanted_parents, record)
280
self._reweave_step('adding inventories')
282
def _setup_steps(self, new_total):
283
"""Setup the markers we need to control the progress bar."""
284
self.total = new_total
287
def _graph_revision(self, rev_id):
288
"""Load a revision into the revision graph."""
289
# pick a random revision
290
# analyse revision id rev_id and put it in the stack.
291
self._reweave_step('loading revisions')
292
rev = self.repo.get_revision_reconcile(rev_id)
294
for parent in rev.parent_ids:
295
if self._parent_is_available(parent):
296
parents.append(parent)
298
mutter('found ghost %s', parent)
299
self._rev_graph[rev_id] = parents
301
def _check_garbage_inventories(self):
302
"""Check for garbage inventories which we cannot trust
304
We cant trust them because their pre-requisite file data may not
305
be present - all we know is that their revision was not installed.
307
if not self.thorough:
309
inventories = set(self.inventory.keys())
310
revisions = set(self.revisions.keys())
311
garbage = inventories.difference(revisions)
312
self.garbage_inventories = len(garbage)
313
for revision_key in garbage:
314
mutter('Garbage inventory {%s} found.', revision_key[-1])
316
def _parent_is_available(self, parent):
317
"""True if parent is a fully available revision
319
A fully available revision has a inventory and a revision object in the
322
if parent in self._rev_graph:
324
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
325
return (inv_present and self.repo.has_revision(parent))
327
def _reweave_step(self, message):
328
"""Mark a single step of regeneration complete."""
329
self.pb.update(message, self.count, self.total)
333
class KnitReconciler(RepoReconciler):
334
"""Reconciler that reconciles a knit format repository.
336
This will detect garbage inventories and remove them in thorough mode.
339
def _reconcile_steps(self):
340
"""Perform the steps to reconcile this repository."""
344
except errors.BzrCheckError:
347
# knits never suffer this
349
self._fix_text_parents()
351
def _load_indexes(self):
352
"""Load indexes for the reconciliation."""
353
self.transaction = self.repo.get_transaction()
354
self.pb.update('Reading indexes.', 0, 2)
355
self.inventory = self.repo.inventories
356
self.pb.update('Reading indexes.', 1, 2)
357
self.repo._check_for_inconsistent_revision_parents()
358
self.revisions = self.repo.revisions
359
self.pb.update('Reading indexes.', 2, 2)
361
def _gc_inventory(self):
362
"""Remove inventories that are not referenced from the revision store."""
363
self.pb.update('Checking unused inventories.', 0, 1)
364
self._check_garbage_inventories()
365
self.pb.update('Checking unused inventories.', 1, 3)
366
if not self.garbage_inventories:
367
self.pb.note('Inventory ok.')
369
self.pb.update('Backing up inventory...', 0, 0)
370
self.repo._backup_inventory()
371
self.pb.note('Backup Inventory created.')
372
# asking for '' should never return a non-empty weave
373
new_inventories = self.repo._temp_inventories()
374
# we have topological order of revisions and non ghost parents ready.
375
graph = self.revisions.get_parent_map(self.revisions.keys())
376
revision_keys = list(TopoSorter(graph).iter_topo_order())
377
revision_ids = [key[-1] for key in revision_keys]
378
self._setup_steps(len(revision_keys))
379
stream = self._change_inv_parents(
380
self.inventory.get_record_stream(revision_keys, 'unsorted', True),
383
new_inventories.insert_record_stream(stream)
384
# if this worked, the set of new_inventory_vf.names should equal
385
# the revisionds list
386
if not(set(new_inventories.keys()) == set(revision_keys)):
387
raise AssertionError()
388
self.pb.update('Writing weave')
389
self.repo._activate_new_inventory()
390
self.inventory = None
391
self.pb.note('Inventory regenerated.')
393
def _fix_text_parents(self):
394
"""Fix bad versionedfile parent entries.
396
It is possible for the parents entry in a versionedfile entry to be
397
inconsistent with the values in the revision and inventory.
399
This method finds entries with such inconsistencies, corrects their
400
parent lists, and replaces the versionedfile with a corrected version.
402
transaction = self.repo.get_transaction()
403
versions = [key[-1] for key in self.revisions.keys()]
404
mutter('Prepopulating revision text cache with %d revisions',
406
vf_checker = self.repo._get_versioned_file_checker()
407
bad_parents, unused_versions = vf_checker.check_file_version_parents(
408
self.repo.texts, self.pb)
409
text_index = vf_checker.text_index
410
per_id_bad_parents = {}
411
for key in unused_versions:
412
# Ensure that every file with unused versions gets rewritten.
413
# NB: This is really not needed, reconcile != pack.
414
per_id_bad_parents[key[0]] = {}
415
# Generate per-knit/weave data.
416
for key, details in bad_parents.iteritems():
419
knit_parents = tuple([parent[-1] for parent in details[0]])
420
correct_parents = tuple([parent[-1] for parent in details[1]])
421
file_details = per_id_bad_parents.setdefault(file_id, {})
422
file_details[rev_id] = (knit_parents, correct_parents)
423
file_id_versions = {}
424
for text_key in text_index:
425
versions_list = file_id_versions.setdefault(text_key[0], [])
426
versions_list.append(text_key[1])
427
# Do the reconcile of individual weaves.
428
for num, file_id in enumerate(per_id_bad_parents):
429
self.pb.update('Fixing text parents', num,
430
len(per_id_bad_parents))
431
versions_with_bad_parents = per_id_bad_parents[file_id]
432
id_unused_versions = set(key[-1] for key in unused_versions
433
if key[0] == file_id)
434
if file_id in file_id_versions:
435
file_versions = file_id_versions[file_id]
437
# This id was present in the disk store but is not referenced
438
# by any revision at all.
440
self._fix_text_parent(file_id, versions_with_bad_parents,
441
id_unused_versions, file_versions)
443
def _fix_text_parent(self, file_id, versions_with_bad_parents,
444
unused_versions, all_versions):
445
"""Fix bad versionedfile entries in a single versioned file."""
446
mutter('fixing text parent: %r (%d versions)', file_id,
447
len(versions_with_bad_parents))
448
mutter('(%d are unused)', len(unused_versions))
449
new_file_id = 'temp:%s' % file_id
452
for version in all_versions:
453
if version in unused_versions:
455
elif version in versions_with_bad_parents:
456
parents = versions_with_bad_parents[version][1]
458
pmap = self.repo.texts.get_parent_map([(file_id, version)])
459
parents = [key[-1] for key in pmap[(file_id, version)]]
460
new_parents[(new_file_id, version)] = [
461
(new_file_id, parent) for parent in parents]
462
needed_keys.add((file_id, version))
463
def fix_parents(stream):
464
for record in stream:
465
bytes = record.get_bytes_as('fulltext')
466
new_key = (new_file_id, record.key[-1])
467
parents = new_parents[new_key]
468
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
469
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
470
self.repo._remove_file_id(new_file_id)
471
self.repo.texts.insert_record_stream(fix_parents(stream))
472
self.repo._remove_file_id(file_id)
474
self.repo._move_file_id(new_file_id, file_id)
477
class PackReconciler(RepoReconciler):
478
"""Reconciler that reconciles a pack based repository.
480
Garbage inventories do not affect ancestry queries, and removal is
481
considerably more expensive as there is no separate versioned file for
482
them, so they are not cleaned. In short it is currently a no-op.
484
In future this may be a good place to hook in annotation cache checking,
485
index recreation etc.
488
# XXX: The index corruption that _fix_text_parents performs is needed for
489
# packs, but not yet implemented. The basic approach is to:
490
# - lock the names list
491
# - perform a customised pack() that regenerates data as needed
492
# - unlock the names list
493
# https://bugs.edge.launchpad.net/bzr/+bug/154173
495
def _reconcile_steps(self):
496
"""Perform the steps to reconcile this repository."""
497
if not self.thorough:
499
collection = self.repo._pack_collection
500
collection.ensure_loaded()
501
collection.lock_names()
503
packs = collection.all_packs()
504
all_revisions = self.repo.all_revision_ids()
505
total_inventories = len(list(
506
collection.inventory_index.combined_index.iter_all_entries()))
507
if len(all_revisions):
508
self._packer = repofmt.pack_repo.ReconcilePacker(
509
collection, packs, ".reconcile", all_revisions)
510
new_pack = self._packer.pack(pb=self.pb)
511
if new_pack is not None:
512
self._discard_and_save(packs)
514
# only make a new pack when there is data to copy.
515
self._discard_and_save(packs)
516
self.garbage_inventories = total_inventories - len(list(
517
collection.inventory_index.combined_index.iter_all_entries()))
519
collection._unlock_names()
521
def _discard_and_save(self, packs):
522
"""Discard some packs from the repository.
524
This removes them from the memory index, saves the in-memory index
525
which makes the newly reconciled pack visible and hides the packs to be
526
discarded, and finally renames the packs being discarded into the
527
obsolete packs directory.
529
:param packs: The packs to discard.
532
self.repo._pack_collection._remove_pack_from_memory(pack)
533
self.repo._pack_collection._save_pack_names()
534
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
