80
76
def _reconcile(self):
81
77
"""Helper function for performing reconciliation."""
82
self._reconcile_branch()
83
self._reconcile_repository()
85
def _reconcile_branch(self):
87
self.branch = self.bzrdir.open_branch()
88
except errors.NotBranchError:
89
# Nothing to check here
90
self.fixed_branch_history = None
92
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
93
branch_reconciler = self.branch.reconcile(thorough=True)
94
self.fixed_branch_history = branch_reconciler.fixed_history
96
def _reconcile_repository(self):
97
78
self.repo = self.bzrdir.find_repository()
98
ui.ui_factory.note('Reconciling repository %s' %
79
self.pb.note('Reconciling repository %s',
80
self.repo.bzrdir.root_transport.base)
100
81
self.pb.update("Reconciling repository", 0, 1)
101
82
repo_reconciler = self.repo.reconcile(thorough=True)
102
83
self.inconsistent_parents = repo_reconciler.inconsistent_parents
103
84
self.garbage_inventories = repo_reconciler.garbage_inventories
104
85
if repo_reconciler.aborted:
106
87
'Reconcile aborted: revision index has inconsistent parents.')
108
89
'Run "bzr check" for more details.')
110
ui.ui_factory.note('Reconciliation complete.')
113
class BranchReconciler(object):
114
"""Reconciler that works on a branch."""
116
def __init__(self, a_branch, thorough=False):
117
self.fixed_history = None
118
self.thorough = thorough
119
self.branch = a_branch
122
operation = cleanup.OperationWithCleanups(self._reconcile)
123
self.add_cleanup = operation.add_cleanup
124
operation.run_simple()
126
def _reconcile(self):
127
self.branch.lock_write()
128
self.add_cleanup(self.branch.unlock)
129
self.pb = ui.ui_factory.nested_progress_bar()
130
self.add_cleanup(self.pb.finished)
131
self._reconcile_steps()
133
def _reconcile_steps(self):
134
self._reconcile_revision_history()
136
def _reconcile_revision_history(self):
137
repo = self.branch.repository
138
last_revno, last_revision_id = self.branch.last_revision_info()
141
for revid in repo.iter_reverse_revision_history(
143
real_history.append(revid)
144
except errors.RevisionNotPresent:
145
pass # Hit a ghost left hand parent
146
real_history.reverse()
147
if last_revno != len(real_history):
148
self.fixed_history = True
149
# Technically for Branch5 formats, it is more efficient to use
150
# set_revision_history, as this will regenerate it again.
151
# Not really worth a whole BranchReconciler class just for this,
153
ui.ui_factory.note('Fixing last revision info %s => %s' % (
154
last_revno, len(real_history)))
155
self.branch.set_last_revision_info(len(real_history),
158
self.fixed_history = False
159
ui.ui_factory.note('revision_history ok.')
91
self.pb.note('Reconciliation complete.')
162
94
class RepoReconciler(object):
163
95
"""Reconciler that reconciles a repository.
165
97
The goal of repository reconciliation is to make any derived data
166
consistent with the core data committed by a user. This can involve
98
consistent with the core data committed by a user. This can involve
167
99
reindexing, or removing unreferenced data if that can interfere with
168
100
queries in a given repository.
234
164
# put a revision into the graph.
235
165
self._graph_revision(rev_id)
236
166
self._check_garbage_inventories()
237
# if there are no inconsistent_parents and
167
# if there are no inconsistent_parents and
238
168
# (no garbage inventories or we are not doing a thorough check)
239
if (not self.inconsistent_parents and
169
if (not self.inconsistent_parents and
240
170
(not self.garbage_inventories or not self.thorough)):
241
ui.ui_factory.note('Inventory ok.')
171
self.pb.note('Inventory ok.')
243
self.pb.update('Backing up inventory', 0, 0)
244
self.repo._backup_inventory()
245
ui.ui_factory.note('Backup inventory created.')
246
new_inventories = self.repo._temp_inventories()
173
self.pb.update('Backing up inventory...', 0, 0)
174
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
175
self.pb.note('Backup Inventory created.')
176
# asking for '' should never return a non-empty weave
177
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
178
self.repo.get_transaction())
248
180
# we have topological order of revisions and non ghost parents ready.
249
181
self._setup_steps(len(self._rev_graph))
250
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
251
stream = self._change_inv_parents(
252
self.inventory.get_record_stream(revision_keys, 'unordered', True),
253
self._new_inv_parents,
255
new_inventories.insert_record_stream(stream)
256
# if this worked, the set of new_inventories.keys should equal
182
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
183
parents = self._rev_graph[rev_id]
184
# double check this really is in topological order.
185
unavailable = [p for p in parents if p not in new_inventory_vf]
186
assert len(unavailable) == 0
187
# this entry has all the non ghost parents in the inventory
189
self._reweave_step('adding inventories')
190
if isinstance(new_inventory_vf, WeaveFile):
191
# It's really a WeaveFile, but we call straight into the
192
# Weave's add method to disable the auto-write-out behaviour.
193
# This is done to avoid a revision_count * time-to-write additional overhead on
195
new_inventory_vf._check_write_ok()
196
Weave._add_lines(new_inventory_vf, rev_id, parents,
197
self.inventory.get_lines(rev_id), None, None, None, False, True)
199
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
201
if isinstance(new_inventory_vf, WeaveFile):
202
new_inventory_vf._save()
203
# if this worked, the set of new_inventory_vf.names should equal
258
if not (set(new_inventories.keys()) ==
259
set([(revid,) for revid in self.pending])):
260
raise AssertionError()
205
assert set(new_inventory_vf.versions()) == self.pending
261
206
self.pb.update('Writing weave')
262
self.repo._activate_new_inventory()
207
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
208
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
263
209
self.inventory = None
264
ui.ui_factory.note('Inventory regenerated.')
266
def _new_inv_parents(self, revision_key):
267
"""Lookup ghost-filtered parents for revision_key."""
268
# Use the filtered ghostless parents list:
269
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
271
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
272
"""Adapt a record stream to reconcile the parents."""
273
for record in stream:
274
wanted_parents = get_parents(record.key)
275
if wanted_parents and wanted_parents[0] not in all_revision_keys:
276
# The check for the left most parent only handles knit
277
# compressors, but this code only applies to knit and weave
278
# repositories anyway.
279
bytes = record.get_bytes_as('fulltext')
280
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
282
adapted_record = AdapterFactory(record.key, wanted_parents, record)
284
self._reweave_step('adding inventories')
210
self.pb.note('Inventory regenerated.')
286
212
def _setup_steps(self, new_total):
287
213
"""Setup the markers we need to control the progress bar."""
294
220
# analyse revision id rev_id and put it in the stack.
295
221
self._reweave_step('loading revisions')
296
222
rev = self.repo.get_revision_reconcile(rev_id)
223
assert rev.revision_id == rev_id
298
225
for parent in rev.parent_ids:
299
226
if self._parent_is_available(parent):
300
227
parents.append(parent)
302
229
mutter('found ghost %s', parent)
303
self._rev_graph[rev_id] = parents
230
self._rev_graph[rev_id] = parents
231
if self._parents_are_inconsistent(rev_id, parents):
232
self.inconsistent_parents += 1
233
mutter('Inconsistent inventory parents: id {%s} '
234
'inventory claims %r, '
235
'available parents are %r, '
236
'unavailable parents are %r',
238
set(self.inventory.get_parents(rev_id)),
240
set(rev.parent_ids).difference(set(parents)))
242
def _parents_are_inconsistent(self, rev_id, parents):
243
"""Return True if the parents list of rev_id does not match the weave.
245
This detects inconsistencies based on the self.thorough value:
246
if thorough is on, the first parent value is checked as well as ghost
248
Otherwise only the ghost differences are evaluated.
250
weave_parents = self.inventory.get_parents(rev_id)
251
weave_missing_old_ghosts = set(weave_parents) != set(parents)
252
first_parent_is_wrong = (
253
len(weave_parents) and len(parents) and
254
parents[0] != weave_parents[0])
256
return weave_missing_old_ghosts or first_parent_is_wrong
258
return weave_missing_old_ghosts
305
260
def _check_garbage_inventories(self):
306
261
"""Check for garbage inventories which we cannot trust
355
308
def _load_indexes(self):
356
309
"""Load indexes for the reconciliation."""
357
310
self.transaction = self.repo.get_transaction()
358
self.pb.update('Reading indexes', 0, 2)
359
self.inventory = self.repo.inventories
360
self.pb.update('Reading indexes', 1, 2)
311
self.pb.update('Reading indexes.', 0, 2)
312
self.inventory = self.repo.get_inventory_weave()
313
self.pb.update('Reading indexes.', 1, 2)
361
314
self.repo._check_for_inconsistent_revision_parents()
362
self.revisions = self.repo.revisions
363
self.pb.update('Reading indexes', 2, 2)
315
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
316
self.pb.update('Reading indexes.', 2, 2)
365
318
def _gc_inventory(self):
366
319
"""Remove inventories that are not referenced from the revision store."""
367
self.pb.update('Checking unused inventories', 0, 1)
320
self.pb.update('Checking unused inventories.', 0, 1)
368
321
self._check_garbage_inventories()
369
self.pb.update('Checking unused inventories', 1, 3)
322
self.pb.update('Checking unused inventories.', 1, 3)
370
323
if not self.garbage_inventories:
371
ui.ui_factory.note('Inventory ok.')
324
self.pb.note('Inventory ok.')
373
self.pb.update('Backing up inventory', 0, 0)
374
self.repo._backup_inventory()
375
ui.ui_factory.note('Backup Inventory created')
326
self.pb.update('Backing up inventory...', 0, 0)
327
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
328
self.pb.note('Backup Inventory created.')
376
329
# asking for '' should never return a non-empty weave
377
new_inventories = self.repo._temp_inventories()
330
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
378
333
# we have topological order of revisions and non ghost parents ready.
379
graph = self.revisions.get_parent_map(self.revisions.keys())
380
revision_keys = topo_sort(graph)
381
revision_ids = [key[-1] for key in revision_keys]
382
self._setup_steps(len(revision_keys))
383
stream = self._change_inv_parents(
384
self.inventory.get_record_stream(revision_keys, 'unordered', True),
387
new_inventories.insert_record_stream(stream)
334
self._setup_steps(len(self.revisions))
335
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
336
parents = self.revisions.get_parents(rev_id)
337
# double check this really is in topological order.
338
unavailable = [p for p in parents if p not in new_inventory_vf]
339
assert len(unavailable) == 0
340
# this entry has all the non ghost parents in the inventory
342
self._reweave_step('adding inventories')
343
# ugly but needed, weaves are just way tooooo slow else.
344
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
388
346
# if this worked, the set of new_inventory_vf.names should equal
389
# the revisionds list
390
if not(set(new_inventories.keys()) == set(revision_keys)):
391
raise AssertionError()
348
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
392
349
self.pb.update('Writing weave')
393
self.repo._activate_new_inventory()
350
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
351
self.repo.control_weaves.delete('inventory.new', self.transaction)
394
352
self.inventory = None
395
ui.ui_factory.note('Inventory regenerated.')
353
self.pb.note('Inventory regenerated.')
355
def _check_garbage_inventories(self):
356
"""Check for garbage inventories which we cannot trust
358
We cant trust them because their pre-requisite file data may not
359
be present - all we know is that their revision was not installed.
361
inventories = set(self.inventory.versions())
362
revisions = set(self.revisions.versions())
363
garbage = inventories.difference(revisions)
364
self.garbage_inventories = len(garbage)
365
for revision_id in garbage:
366
mutter('Garbage inventory {%s} found.', revision_id)
397
368
def _fix_text_parents(self):
398
369
"""Fix bad versionedfile parent entries.
404
375
parent lists, and replaces the versionedfile with a corrected version.
406
377
transaction = self.repo.get_transaction()
407
versions = [key[-1] for key in self.revisions.keys()]
378
revision_versions = repository._RevisionTextVersionCache(self.repo)
379
versions = self.revisions.versions()
408
380
mutter('Prepopulating revision text cache with %d revisions',
410
vf_checker = self.repo._get_versioned_file_checker()
411
bad_parents, unused_versions = vf_checker.check_file_version_parents(
412
self.repo.texts, self.pb)
413
text_index = vf_checker.text_index
414
per_id_bad_parents = {}
415
for key in unused_versions:
416
# Ensure that every file with unused versions gets rewritten.
417
# NB: This is really not needed, reconcile != pack.
418
per_id_bad_parents[key[0]] = {}
419
# Generate per-knit/weave data.
420
for key, details in bad_parents.iteritems():
423
knit_parents = tuple([parent[-1] for parent in details[0]])
424
correct_parents = tuple([parent[-1] for parent in details[1]])
425
file_details = per_id_bad_parents.setdefault(file_id, {})
426
file_details[rev_id] = (knit_parents, correct_parents)
427
file_id_versions = {}
428
for text_key in text_index:
429
versions_list = file_id_versions.setdefault(text_key[0], [])
430
versions_list.append(text_key[1])
431
# Do the reconcile of individual weaves.
432
for num, file_id in enumerate(per_id_bad_parents):
382
revision_versions.prepopulate_revs(versions)
383
used_file_versions = revision_versions.used_file_versions()
384
for num, file_id in enumerate(self.repo.weave_store):
433
385
self.pb.update('Fixing text parents', num,
434
len(per_id_bad_parents))
435
versions_with_bad_parents = per_id_bad_parents[file_id]
436
id_unused_versions = set(key[-1] for key in unused_versions
437
if key[0] == file_id)
438
if file_id in file_id_versions:
439
file_versions = file_id_versions[file_id]
441
# This id was present in the disk store but is not referenced
442
# by any revision at all.
444
self._fix_text_parent(file_id, versions_with_bad_parents,
445
id_unused_versions, file_versions)
386
len(self.repo.weave_store))
387
vf = self.repo.weave_store.get_weave(file_id, transaction)
388
vf_checker = self.repo.get_versioned_file_checker(
389
vf.versions(), revision_versions)
390
versions_with_bad_parents, dangling_file_versions = \
391
vf_checker.check_file_version_parents(vf, file_id)
392
if (len(versions_with_bad_parents) == 0 and
393
len(dangling_file_versions) == 0):
395
full_text_versions = set()
396
unused_versions = set()
397
for dangling_version in dangling_file_versions:
398
version = dangling_version[1]
399
if dangling_version in used_file_versions:
400
# This version *is* used by some revision, even though it
401
# isn't used by its own revision! We make sure any
402
# revision referencing it is stored as a fulltext
403
# This avoids bug 155730: it means that clients looking at
404
# inventories to determine the versions to fetch will not
405
# miss a required version. (So clients can assume that if
406
# they have a complete revision graph, and fetch all file
407
# versions named by those revisions inventories, then they
408
# will not have any missing parents for 'delta' knit
410
# XXX: A better, but more difficult and slower fix would be
411
# to rewrite the inventories referencing this version.
412
full_text_versions.add(version)
414
# This version is totally unreferenced. It should be
416
unused_versions.add(version)
417
self._fix_text_parent(file_id, vf, versions_with_bad_parents,
418
full_text_versions, unused_versions)
447
def _fix_text_parent(self, file_id, versions_with_bad_parents,
448
unused_versions, all_versions):
420
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents,
421
full_text_versions, unused_versions):
449
422
"""Fix bad versionedfile entries in a single versioned file."""
450
423
mutter('fixing text parent: %r (%d versions)', file_id,
451
424
len(versions_with_bad_parents))
452
mutter('(%d are unused)', len(unused_versions))
453
new_file_id = 'temp:%s' % file_id
425
mutter('(%d need to be full texts, %d are unused)',
426
len(full_text_versions), len(unused_versions))
427
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
456
for version in all_versions:
430
for version in vf.versions():
431
if version in versions_with_bad_parents:
432
parents = versions_with_bad_parents[version][1]
434
parents = vf.get_parents(version)
435
new_parents[version] = parents
436
for version in TopoSorter(new_parents.items()).iter_topo_order():
457
437
if version in unused_versions:
459
elif version in versions_with_bad_parents:
460
parents = versions_with_bad_parents[version][1]
439
lines = vf.get_lines(version)
440
parents = new_parents[version]
441
if parents and (parents[0] in full_text_versions):
442
# Force this record to be a fulltext, not a delta.
443
new_vf._add(version, lines, parents, False,
444
None, None, None, False)
462
pmap = self.repo.texts.get_parent_map([(file_id, version)])
463
parents = [key[-1] for key in pmap[(file_id, version)]]
464
new_parents[(new_file_id, version)] = [
465
(new_file_id, parent) for parent in parents]
466
needed_keys.add((file_id, version))
467
def fix_parents(stream):
468
for record in stream:
469
bytes = record.get_bytes_as('fulltext')
470
new_key = (new_file_id, record.key[-1])
471
parents = new_parents[new_key]
472
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
473
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
474
self.repo._remove_file_id(new_file_id)
475
self.repo.texts.insert_record_stream(fix_parents(stream))
476
self.repo._remove_file_id(file_id)
478
self.repo._move_file_id(new_file_id, file_id)
446
new_vf.add_lines(version, parents, lines)
447
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
448
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
481
451
class PackReconciler(RepoReconciler):
499
469
def _reconcile_steps(self):
500
470
"""Perform the steps to reconcile this repository."""
501
if not self.thorough:
503
collection = self.repo._pack_collection
504
collection.ensure_loaded()
505
collection.lock_names()
506
self.add_cleanup(collection._unlock_names)
507
packs = collection.all_packs()
508
all_revisions = self.repo.all_revision_ids()
509
total_inventories = len(list(
510
collection.inventory_index.combined_index.iter_all_entries()))
511
if len(all_revisions):
512
new_pack = self.repo._reconcile_pack(collection, packs,
513
".reconcile", all_revisions, self.pb)
514
if new_pack is not None:
515
self._discard_and_save(packs)
517
# only make a new pack when there is data to copy.
518
self._discard_and_save(packs)
519
self.garbage_inventories = total_inventories - len(list(
520
collection.inventory_index.combined_index.iter_all_entries()))
522
def _discard_and_save(self, packs):
523
"""Discard some packs from the repository.
525
This removes them from the memory index, saves the in-memory index
526
which makes the newly reconciled pack visible and hides the packs to be
527
discarded, and finally renames the packs being discarded into the
528
obsolete packs directory.
530
:param packs: The packs to discard.
533
self.repo._pack_collection._remove_pack_from_memory(pack)
534
self.repo._pack_collection._save_pack_names()
535
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
