/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/reconcile.py

  • Committer: Jelmer Vernooij
  • Date: 2018-11-21 03:39:28 UTC
  • mto: This revision was merged to the branch mainline in revision 7206.
  • Revision ID: jelmer@jelmer.uk-20181121033928-ck4sb5zfdwosw35b
Fix test.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/reconcile.py
16
16
 
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
18
18
 
 
19
from __future__ import absolute_import
19
20
 
20
21
__all__ = [
21
22
    'KnitReconciler',
26
27
    ]
27
28
 
28
29
 
29
 
from bzrlib import (
 
30
from . import (
30
31
    cleanup,
31
32
    errors,
 
33
    revision as _mod_revision,
32
34
    ui,
33
35
    )
34
 
from bzrlib.trace import mutter
35
 
from bzrlib.tsort import topo_sort
36
 
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
37
 
 
38
 
 
39
 
def reconcile(dir, other=None):
 
36
from .trace import mutter
 
37
from .tsort import topo_sort
 
38
from .bzr.versionedfile import AdapterFactory, FulltextContentFactory
 
39
from .i18n import gettext
 
40
 
 
41
 
 
42
def reconcile(dir, canonicalize_chks=False):
40
43
    """Reconcile the data in dir.
41
44
 
42
45
    Currently this is limited to a inventory 'reweave'.
46
49
    Directly using Reconciler is recommended for library users that
47
50
    desire fine grained control or analysis of the found issues.
48
51
 
49
 
    :param other: another bzrdir to reconcile against.
 
52
    :param canonicalize_chks: Make sure CHKs are in canonical form.
50
53
    """
51
 
    reconciler = Reconciler(dir, other=other)
 
54
    reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
52
55
    reconciler.reconcile()
53
56
 
54
57
 
55
58
class Reconciler(object):
56
59
    """Reconcilers are used to reconcile existing data."""
57
60
 
58
 
    def __init__(self, dir, other=None):
 
61
    def __init__(self, dir, other=None, canonicalize_chks=False):
59
62
        """Create a Reconciler."""
60
 
        self.bzrdir = dir
 
63
        self.controldir = dir
 
64
        self.canonicalize_chks = canonicalize_chks
61
65
 
62
66
    def reconcile(self):
63
67
        """Perform reconciliation.
64
68
 
65
69
        After reconciliation the following attributes document found issues:
66
 
        inconsistent_parents: The number of revisions in the repository whose
67
 
                              ancestry was being reported incorrectly.
68
 
        garbage_inventories: The number of inventory objects without revisions
69
 
                             that were garbage collected.
70
 
        fixed_branch_history: None if there was no branch, False if the branch
71
 
                              history was correct, True if the branch history
72
 
                              needed to be re-normalized.
 
70
 
 
71
        * `inconsistent_parents`: The number of revisions in the repository
 
72
          whose ancestry was being reported incorrectly.
 
73
        * `garbage_inventories`: The number of inventory objects without
 
74
          revisions that were garbage collected.
 
75
        * `fixed_branch_history`: None if there was no branch, False if the
 
76
          branch history was correct, True if the branch history needed to be
 
77
          re-normalized.
73
78
        """
74
 
        self.pb = ui.ui_factory.nested_progress_bar()
75
 
        try:
76
 
            self._reconcile()
77
 
        finally:
78
 
            self.pb.finished()
 
79
        operation = cleanup.OperationWithCleanups(self._reconcile)
 
80
        self.add_cleanup = operation.add_cleanup
 
81
        operation.run_simple()
79
82
 
80
83
    def _reconcile(self):
81
84
        """Helper function for performing reconciliation."""
 
85
        self.pb = ui.ui_factory.nested_progress_bar()
 
86
        self.add_cleanup(self.pb.finished)
82
87
        self._reconcile_branch()
83
88
        self._reconcile_repository()
84
89
 
85
90
    def _reconcile_branch(self):
86
91
        try:
87
 
            self.branch = self.bzrdir.open_branch()
 
92
            self.branch = self.controldir.open_branch()
88
93
        except errors.NotBranchError:
89
94
            # Nothing to check here
90
95
            self.fixed_branch_history = None
91
96
            return
92
 
        ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
 
97
        ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
93
98
        branch_reconciler = self.branch.reconcile(thorough=True)
94
99
        self.fixed_branch_history = branch_reconciler.fixed_history
95
100
 
96
101
    def _reconcile_repository(self):
97
 
        self.repo = self.bzrdir.find_repository()
98
 
        ui.ui_factory.note('Reconciling repository %s' %
99
 
            self.repo.user_url)
100
 
        self.pb.update("Reconciling repository", 0, 1)
101
 
        repo_reconciler = self.repo.reconcile(thorough=True)
 
102
        self.repo = self.controldir.find_repository()
 
103
        ui.ui_factory.note(gettext('Reconciling repository %s') %
 
104
                           self.repo.user_url)
 
105
        self.pb.update(gettext("Reconciling repository"), 0, 1)
 
106
        if self.canonicalize_chks:
 
107
            try:
 
108
                self.repo.reconcile_canonicalize_chks
 
109
            except AttributeError:
 
110
                raise errors.BzrError(
 
111
                    gettext("%s cannot canonicalize CHKs.") % (self.repo,))
 
112
            repo_reconciler = self.repo.reconcile_canonicalize_chks()
 
113
        else:
 
114
            repo_reconciler = self.repo.reconcile(thorough=True)
102
115
        self.inconsistent_parents = repo_reconciler.inconsistent_parents
103
116
        self.garbage_inventories = repo_reconciler.garbage_inventories
104
117
        if repo_reconciler.aborted:
105
 
            ui.ui_factory.note(
106
 
                'Reconcile aborted: revision index has inconsistent parents.')
107
 
            ui.ui_factory.note(
108
 
                'Run "bzr check" for more details.')
 
118
            ui.ui_factory.note(gettext(
 
119
                'Reconcile aborted: revision index has inconsistent parents.'))
 
120
            ui.ui_factory.note(gettext(
 
121
                'Run "brz check" for more details.'))
109
122
        else:
110
 
            ui.ui_factory.note('Reconciliation complete.')
 
123
            ui.ui_factory.note(gettext('Reconciliation complete.'))
111
124
 
112
125
 
113
126
class BranchReconciler(object):
134
147
        self._reconcile_revision_history()
135
148
 
136
149
    def _reconcile_revision_history(self):
137
 
        repo = self.branch.repository
138
150
        last_revno, last_revision_id = self.branch.last_revision_info()
139
151
        real_history = []
 
152
        graph = self.branch.repository.get_graph()
140
153
        try:
141
 
            for revid in repo.iter_reverse_revision_history(
142
 
                    last_revision_id):
 
154
            for revid in graph.iter_lefthand_ancestry(
 
155
                    last_revision_id, (_mod_revision.NULL_REVISION,)):
143
156
                real_history.append(revid)
144
157
        except errors.RevisionNotPresent:
145
 
            pass # Hit a ghost left hand parent
 
158
            pass  # Hit a ghost left hand parent
146
159
        real_history.reverse()
147
160
        if last_revno != len(real_history):
148
161
            self.fixed_history = True
150
163
            # set_revision_history, as this will regenerate it again.
151
164
            # Not really worth a whole BranchReconciler class just for this,
152
165
            # though.
153
 
            ui.ui_factory.note('Fixing last revision info %s => %s' % (
154
 
                 last_revno, len(real_history)))
 
166
            ui.ui_factory.note(gettext('Fixing last revision info {0} '
 
167
                                       ' => {1}').format(
 
168
                last_revno, len(real_history)))
155
169
            self.branch.set_last_revision_info(len(real_history),
156
170
                                               last_revision_id)
157
171
        else:
158
172
            self.fixed_history = False
159
 
            ui.ui_factory.note('revision_history ok.')
 
173
            ui.ui_factory.note(gettext('revision_history ok.'))
160
174
 
161
175
 
162
176
class RepoReconciler(object):
187
201
        """Perform reconciliation.
188
202
 
189
203
        After reconciliation the following attributes document found issues:
190
 
        inconsistent_parents: The number of revisions in the repository whose
191
 
                              ancestry was being reported incorrectly.
192
 
        garbage_inventories: The number of inventory objects without revisions
193
 
                             that were garbage collected.
 
204
 
 
205
        * `inconsistent_parents`: The number of revisions in the repository
 
206
          whose ancestry was being reported incorrectly.
 
207
        * `garbage_inventories`: The number of inventory objects without
 
208
          revisions that were garbage collected.
194
209
        """
195
210
        operation = cleanup.OperationWithCleanups(self._reconcile)
196
211
        self.add_cleanup = operation.add_cleanup
215
230
        only data-loss causing issues (!self.thorough) or all issues
216
231
        (self.thorough) are treated as requiring the reweave.
217
232
        """
218
 
        # local because needing to know about WeaveFile is a wart we want to hide
219
 
        from bzrlib.weave import WeaveFile, Weave
220
233
        transaction = self.repo.get_transaction()
221
 
        self.pb.update('Reading inventory data')
 
234
        self.pb.update(gettext('Reading inventory data'))
222
235
        self.inventory = self.repo.inventories
223
236
        self.revisions = self.repo.revisions
224
237
        # the total set of revisions to process
225
 
        self.pending = set([key[-1] for key in self.revisions.keys()])
 
238
        self.pending = {key[-1] for key in self.revisions.keys()}
226
239
 
227
240
        # mapping from revision_id to parents
228
241
        self._rev_graph = {}
236
249
        self._check_garbage_inventories()
237
250
        # if there are no inconsistent_parents and
238
251
        # (no garbage inventories or we are not doing a thorough check)
239
 
        if (not self.inconsistent_parents and
240
 
            (not self.garbage_inventories or not self.thorough)):
241
 
            ui.ui_factory.note('Inventory ok.')
 
252
        if (not self.inconsistent_parents
 
253
                and (not self.garbage_inventories or not self.thorough)):
 
254
            ui.ui_factory.note(gettext('Inventory ok.'))
242
255
            return
243
 
        self.pb.update('Backing up inventory', 0, 0)
 
256
        self.pb.update(gettext('Backing up inventory'), 0, 0)
244
257
        self.repo._backup_inventory()
245
 
        ui.ui_factory.note('Backup inventory created.')
 
258
        ui.ui_factory.note(gettext('Backup inventory created.'))
246
259
        new_inventories = self.repo._temp_inventories()
247
260
 
248
261
        # we have topological order of revisions and non ghost parents ready.
255
268
        new_inventories.insert_record_stream(stream)
256
269
        # if this worked, the set of new_inventories.keys should equal
257
270
        # self.pending
258
 
        if not (set(new_inventories.keys()) ==
259
 
            set([(revid,) for revid in self.pending])):
 
271
        if not (set(new_inventories.keys())
 
272
                == {(revid,) for revid in self.pending}):
260
273
            raise AssertionError()
261
 
        self.pb.update('Writing weave')
 
274
        self.pb.update(gettext('Writing weave'))
262
275
        self.repo._activate_new_inventory()
263
276
        self.inventory = None
264
 
        ui.ui_factory.note('Inventory regenerated.')
 
277
        ui.ui_factory.note(gettext('Inventory regenerated.'))
265
278
 
266
279
    def _new_inv_parents(self, revision_key):
267
280
        """Lookup ghost-filtered parents for revision_key."""
279
292
                bytes = record.get_bytes_as('fulltext')
280
293
                yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
281
294
            else:
282
 
                adapted_record = AdapterFactory(record.key, wanted_parents, record)
 
295
                adapted_record = AdapterFactory(
 
296
                    record.key, wanted_parents, record)
283
297
                yield adapted_record
284
298
            self._reweave_step('adding inventories')
285
299
 
355
369
    def _load_indexes(self):
356
370
        """Load indexes for the reconciliation."""
357
371
        self.transaction = self.repo.get_transaction()
358
 
        self.pb.update('Reading indexes', 0, 2)
 
372
        self.pb.update(gettext('Reading indexes'), 0, 2)
359
373
        self.inventory = self.repo.inventories
360
 
        self.pb.update('Reading indexes', 1, 2)
 
374
        self.pb.update(gettext('Reading indexes'), 1, 2)
361
375
        self.repo._check_for_inconsistent_revision_parents()
362
376
        self.revisions = self.repo.revisions
363
 
        self.pb.update('Reading indexes', 2, 2)
 
377
        self.pb.update(gettext('Reading indexes'), 2, 2)
364
378
 
365
379
    def _gc_inventory(self):
366
380
        """Remove inventories that are not referenced from the revision store."""
367
 
        self.pb.update('Checking unused inventories', 0, 1)
 
381
        self.pb.update(gettext('Checking unused inventories'), 0, 1)
368
382
        self._check_garbage_inventories()
369
 
        self.pb.update('Checking unused inventories', 1, 3)
 
383
        self.pb.update(gettext('Checking unused inventories'), 1, 3)
370
384
        if not self.garbage_inventories:
371
 
            ui.ui_factory.note('Inventory ok.')
 
385
            ui.ui_factory.note(gettext('Inventory ok.'))
372
386
            return
373
 
        self.pb.update('Backing up inventory', 0, 0)
 
387
        self.pb.update(gettext('Backing up inventory'), 0, 0)
374
388
        self.repo._backup_inventory()
375
 
        ui.ui_factory.note('Backup Inventory created')
 
389
        ui.ui_factory.note(gettext('Backup Inventory created'))
376
390
        # asking for '' should never return a non-empty weave
377
391
        new_inventories = self.repo._temp_inventories()
378
392
        # we have topological order of revisions and non ghost parents ready.
389
403
        # the revisionds list
390
404
        if not(set(new_inventories.keys()) == set(revision_keys)):
391
405
            raise AssertionError()
392
 
        self.pb.update('Writing weave')
 
406
        self.pb.update(gettext('Writing weave'))
393
407
        self.repo._activate_new_inventory()
394
408
        self.inventory = None
395
 
        ui.ui_factory.note('Inventory regenerated.')
 
409
        ui.ui_factory.note(gettext('Inventory regenerated.'))
396
410
 
397
411
    def _fix_text_parents(self):
398
412
        """Fix bad versionedfile parent entries.
406
420
        transaction = self.repo.get_transaction()
407
421
        versions = [key[-1] for key in self.revisions.keys()]
408
422
        mutter('Prepopulating revision text cache with %d revisions',
409
 
                len(versions))
 
423
               len(versions))
410
424
        vf_checker = self.repo._get_versioned_file_checker()
411
425
        bad_parents, unused_versions = vf_checker.check_file_version_parents(
412
426
            self.repo.texts, self.pb)
417
431
            # NB: This is really not needed, reconcile != pack.
418
432
            per_id_bad_parents[key[0]] = {}
419
433
        # Generate per-knit/weave data.
420
 
        for key, details in bad_parents.iteritems():
 
434
        for key, details in bad_parents.items():
421
435
            file_id = key[0]
422
436
            rev_id = key[1]
423
437
            knit_parents = tuple([parent[-1] for parent in details[0]])
430
444
            versions_list.append(text_key[1])
431
445
        # Do the reconcile of individual weaves.
432
446
        for num, file_id in enumerate(per_id_bad_parents):
433
 
            self.pb.update('Fixing text parents', num,
 
447
            self.pb.update(gettext('Fixing text parents'), num,
434
448
                           len(per_id_bad_parents))
435
449
            versions_with_bad_parents = per_id_bad_parents[file_id]
436
450
            id_unused_versions = set(key[-1] for key in unused_versions
437
 
                if key[0] == file_id)
 
451
                                     if key[0] == file_id)
438
452
            if file_id in file_id_versions:
439
453
                file_versions = file_id_versions[file_id]
440
454
            else:
442
456
                # by any revision at all.
443
457
                file_versions = []
444
458
            self._fix_text_parent(file_id, versions_with_bad_parents,
445
 
                 id_unused_versions, file_versions)
 
459
                                  id_unused_versions, file_versions)
446
460
 
447
461
    def _fix_text_parent(self, file_id, versions_with_bad_parents,
448
 
            unused_versions, all_versions):
 
462
                         unused_versions, all_versions):
449
463
        """Fix bad versionedfile entries in a single versioned file."""
450
464
        mutter('fixing text parent: %r (%d versions)', file_id,
451
 
                len(versions_with_bad_parents))
 
465
               len(versions_with_bad_parents))
452
466
        mutter('(%d are unused)', len(unused_versions))
453
 
        new_file_id = 'temp:%s' % file_id
 
467
        new_file_id = b'temp:%s' % file_id
454
468
        new_parents = {}
455
469
        needed_keys = set()
456
470
        for version in all_versions:
464
478
            new_parents[(new_file_id, version)] = [
465
479
                (new_file_id, parent) for parent in parents]
466
480
            needed_keys.add((file_id, version))
 
481
 
467
482
        def fix_parents(stream):
468
483
            for record in stream:
469
484
                bytes = record.get_bytes_as('fulltext')
470
485
                new_key = (new_file_id, record.key[-1])
471
486
                parents = new_parents[new_key]
472
487
                yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
473
 
        stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
 
488
        stream = self.repo.texts.get_record_stream(
 
489
            needed_keys, 'topological', True)
474
490
        self.repo._remove_file_id(new_file_id)
475
491
        self.repo.texts.insert_record_stream(fix_parents(stream))
476
492
        self.repo._remove_file_id(file_id)
494
510
    #  - lock the names list
495
511
    #  - perform a customised pack() that regenerates data as needed
496
512
    #  - unlock the names list
497
 
    # https://bugs.edge.launchpad.net/bzr/+bug/154173
 
513
    # https://bugs.launchpad.net/bzr/+bug/154173
 
514
 
 
515
    def __init__(self, repo, other=None, thorough=False,
 
516
                 canonicalize_chks=False):
 
517
        super(PackReconciler, self).__init__(repo, other=other,
 
518
                                             thorough=thorough)
 
519
        self.canonicalize_chks = canonicalize_chks
498
520
 
499
521
    def _reconcile_steps(self):
500
522
        """Perform the steps to reconcile this repository."""
509
531
        total_inventories = len(list(
510
532
            collection.inventory_index.combined_index.iter_all_entries()))
511
533
        if len(all_revisions):
512
 
            new_pack =  self.repo._reconcile_pack(collection, packs,
513
 
                ".reconcile", all_revisions, self.pb)
 
534
            if self.canonicalize_chks:
 
535
                reconcile_meth = self.repo._canonicalize_chks_pack
 
536
            else:
 
537
                reconcile_meth = self.repo._reconcile_pack
 
538
            new_pack = reconcile_meth(collection, packs, ".reconcile",
 
539
                                      all_revisions, self.pb)
514
540
            if new_pack is not None:
515
541
                self._discard_and_save(packs)
516
542
        else: