/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Martin Pool
  • Date: 2009-07-27 06:28:35 UTC
  • mto: This revision was merged to the branch mainline in revision 4587.
  • Revision ID: mbp@sourcefrog.net-20090727062835-o66p8it658tq1sma
Add CountedLock.get_physical_lock_status

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008-2011 Canonical Ltd
 
1
# Copyright (C) 2008, 2009 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
16
16
 
17
17
"""Repository formats using CHK inventories and groupcompress compression."""
18
18
 
19
 
from __future__ import absolute_import
20
 
 
21
19
import time
22
20
 
23
 
from .. import (
24
 
    controldir,
 
21
from bzrlib import (
 
22
    bzrdir,
 
23
    chk_map,
 
24
    chk_serializer,
25
25
    debug,
26
26
    errors,
 
27
    index as _mod_index,
 
28
    inventory,
 
29
    knit,
27
30
    osutils,
 
31
    pack,
 
32
    remote,
28
33
    revision as _mod_revision,
29
34
    trace,
30
35
    ui,
31
36
    )
32
 
from ..bzr import (
33
 
    chk_map,
34
 
    chk_serializer,
35
 
    index as _mod_index,
36
 
    inventory,
37
 
    pack,
38
 
    versionedfile,
39
 
    )
40
 
from ..bzr.btree_index import (
 
37
from bzrlib.btree_index import (
41
38
    BTreeGraphIndex,
42
39
    BTreeBuilder,
43
40
    )
44
 
from ..bzr.groupcompress import (
 
41
from bzrlib.groupcompress import (
45
42
    _GCGraphIndex,
46
43
    GroupCompressVersionedFiles,
47
44
    )
48
 
from .pack_repo import (
49
 
    _DirectPackAccess,
 
45
from bzrlib.repofmt.pack_repo import (
50
46
    Pack,
51
47
    NewPack,
52
 
    PackRepository,
53
 
    PackCommitBuilder,
 
48
    KnitPackRepository,
 
49
    KnitPackStreamSource,
 
50
    PackRootCommitBuilder,
54
51
    RepositoryPackCollection,
55
52
    RepositoryFormatPack,
56
53
    ResumedPack,
57
54
    Packer,
58
55
    )
59
 
from ..bzr.vf_repository import (
60
 
    StreamSource,
61
 
    )
62
 
from ..sixish import (
63
 
    viewitems,
64
 
    viewvalues,
65
 
    )
66
 
from ..static_tuple import StaticTuple
67
56
 
68
57
 
69
58
class GCPack(NewPack):
91
80
        else:
92
81
            chk_index = None
93
82
        Pack.__init__(self,
94
 
                      # Revisions: parents list, no text compression.
95
 
                      index_builder_class(reference_lists=1),
96
 
                      # Inventory: We want to map compression only, but currently the
97
 
                      # knit code hasn't been updated enough to understand that, so we
98
 
                      # have a regular 2-list index giving parents and compression
99
 
                      # source.
100
 
                      index_builder_class(reference_lists=1),
101
 
                      # Texts: per file graph, for all fileids - so one reference list
102
 
                      # and two elements in the key tuple.
103
 
                      index_builder_class(reference_lists=1, key_elements=2),
104
 
                      # Signatures: Just blobs to store, no compression, no parents
105
 
                      # listing.
106
 
                      index_builder_class(reference_lists=0),
107
 
                      # CHK based storage - just blobs, no compression or parents.
108
 
                      chk_index=chk_index
109
 
                      )
 
83
            # Revisions: parents list, no text compression.
 
84
            index_builder_class(reference_lists=1),
 
85
            # Inventory: We want to map compression only, but currently the
 
86
            # knit code hasn't been updated enough to understand that, so we
 
87
            # have a regular 2-list index giving parents and compression
 
88
            # source.
 
89
            index_builder_class(reference_lists=1),
 
90
            # Texts: per file graph, for all fileids - so one reference list
 
91
            # and two elements in the key tuple.
 
92
            index_builder_class(reference_lists=1, key_elements=2),
 
93
            # Signatures: Just blobs to store, no compression, no parents
 
94
            # listing.
 
95
            index_builder_class(reference_lists=0),
 
96
            # CHK based storage - just blobs, no compression or parents.
 
97
            chk_index=chk_index
 
98
            )
110
99
        self._pack_collection = pack_collection
111
100
        # When we make readonly indices, we need this.
112
101
        self.index_class = pack_collection._index_class
137
126
            self.random_name, mode=self._file_mode)
138
127
        if 'pack' in debug.debug_flags:
139
128
            trace.mutter('%s: create_pack: pack stream open: %s%s t+%6.3fs',
140
 
                         time.ctime(), self.upload_transport.base, self.random_name,
141
 
                         time.time() - self.start_time)
 
129
                time.ctime(), self.upload_transport.base, self.random_name,
 
130
                time.time() - self.start_time)
142
131
        # A list of byte sequences to be written to the new pack, and the
143
132
        # aggregate size of them.  Stored as a list rather than separate
144
133
        # variables so that the _write_data closure below can update them.
148
137
        # robertc says- this is a closure rather than a method on the object
149
138
        # so that the variables are locals, and faster than accessing object
150
139
        # members.
151
 
 
152
 
        def _write_data(data, flush=False, _buffer=self._buffer,
153
 
                        _write=self.write_stream.write, _update=self._hash.update):
154
 
            _buffer[0].append(data)
155
 
            _buffer[1] += len(data)
 
140
        def _write_data(bytes, flush=False, _buffer=self._buffer,
 
141
            _write=self.write_stream.write, _update=self._hash.update):
 
142
            _buffer[0].append(bytes)
 
143
            _buffer[1] += len(bytes)
156
144
            # buffer cap
157
145
            if _buffer[1] > self._cache_limit or flush:
158
 
                data = b''.join(_buffer[0])
159
 
                _write(data)
160
 
                _update(data)
 
146
                bytes = ''.join(_buffer[0])
 
147
                _write(bytes)
 
148
                _update(bytes)
161
149
                _buffer[:] = [[], 0]
162
150
        # expose this on self, for the occasion when clients want to add data.
163
151
        self._write_data = _write_data
166
154
        self._writer.begin()
167
155
        # what state is the pack in? (open, finished, aborted)
168
156
        self._state = 'open'
169
 
        # no name until we finish writing the content
170
 
        self.name = None
171
157
 
172
158
    def _check_references(self):
173
159
        """Make sure our external references are present.
205
191
        self._pack_collection = pack_collection
206
192
        # ATM, We only support this for GCCHK repositories
207
193
        if pack_collection.chk_index is None:
208
 
            raise AssertionError(
209
 
                'pack_collection.chk_index should not be None')
 
194
            raise AssertionError('pack_collection.chk_index should not be None')
210
195
        self._gather_text_refs = False
211
196
        self._chk_id_roots = []
212
197
        self._chk_p_id_roots = []
216
201
 
217
202
    def _get_progress_stream(self, source_vf, keys, message, pb):
218
203
        def pb_stream():
219
 
            substream = source_vf.get_record_stream(
220
 
                keys, 'groupcompress', True)
 
204
            substream = source_vf.get_record_stream(keys, 'groupcompress', True)
221
205
            for idx, record in enumerate(substream):
222
206
                if pb is not None:
223
207
                    pb.update(message, idx + 1, len(keys))
227
211
    def _get_filtered_inv_stream(self, source_vf, keys, message, pb=None):
228
212
        """Filter the texts of inventories, to find the chk pages."""
229
213
        total_keys = len(keys)
230
 
 
231
214
        def _filtered_inv_stream():
232
215
            id_roots_set = set()
233
216
            p_id_roots_set = set()
234
217
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
235
218
            for idx, record in enumerate(stream):
236
219
                # Inventories should always be with revisions; assume success.
237
 
                lines = record.get_bytes_as('lines')
238
 
                chk_inv = inventory.CHKInventory.deserialise(
239
 
                    None, lines, record.key)
 
220
                bytes = record.get_bytes_as('fulltext')
 
221
                chk_inv = inventory.CHKInventory.deserialise(None, bytes,
 
222
                                                             record.key)
240
223
                if pb is not None:
241
224
                    pb.update('inv', idx, total_keys)
242
225
                key = chk_inv.id_to_entry.key()
277
260
        remaining_keys = set(keys)
278
261
        counter = [0]
279
262
        if self._gather_text_refs:
 
263
            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
280
264
            self._text_refs = set()
281
 
 
282
265
        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):
283
266
            cur_keys = root_keys
284
267
            while cur_keys:
285
268
                keys_by_search_prefix = {}
286
269
                remaining_keys.difference_update(cur_keys)
287
270
                next_keys = set()
288
 
 
289
271
                def handle_internal_node(node):
290
 
                    for prefix, value in viewitems(node._items):
 
272
                    for prefix, value in node._items.iteritems():
291
273
                        # We don't want to request the same key twice, and we
292
274
                        # want to order it by the first time it is seen.
293
275
                        # Even further, we don't want to request a key which is
299
281
                        #       always fill them in for stacked branches
300
282
                        if value not in next_keys and value in remaining_keys:
301
283
                            keys_by_search_prefix.setdefault(prefix,
302
 
                                                             []).append(value)
 
284
                                []).append(value)
303
285
                            next_keys.add(value)
304
 
 
305
286
                def handle_leaf_node(node):
306
287
                    # Store is None, because we know we have a LeafNode, and we
307
288
                    # just want its entries
308
289
                    for file_id, bytes in node.iteritems(None):
309
 
                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))
310
 
 
 
290
                        name_utf8, file_id, revision_id = bytes_to_info(bytes)
 
291
                        self._text_refs.add((file_id, revision_id))
311
292
                def next_stream():
312
293
                    stream = source_vf.get_record_stream(cur_keys,
313
294
                                                         'as-requested', True)
369
350
        """Build a VersionedFiles instance on top of this group of packs."""
370
351
        index_name = index_name + '_index'
371
352
        index_to_pack = {}
372
 
        access = _DirectPackAccess(index_to_pack,
373
 
                                   reload_func=self._reload_func)
 
353
        access = knit._DirectPackAccess(index_to_pack)
374
354
        if for_write:
375
355
            # Use new_pack
376
356
            if self.new_pack is None:
410
390
                     pb_offset):
411
391
        trace.mutter('repacking %d %s', len(keys), message)
412
392
        self.pb.update('repacking %s' % (message,), pb_offset)
413
 
        with ui.ui_factory.nested_progress_bar() as child_pb:
 
393
        child_pb = ui.ui_factory.nested_progress_bar()
 
394
        try:
414
395
            stream = vf_to_stream(source_vf, keys, message, child_pb)
415
 
            for _, _ in target_vf._insert_record_stream(
416
 
                    stream, random_id=True, reuse_blocks=False):
 
396
            for _ in target_vf._insert_record_stream(stream,
 
397
                                                     random_id=True,
 
398
                                                     reuse_blocks=False):
417
399
                pass
 
400
        finally:
 
401
            child_pb.finished()
418
402
 
419
403
    def _copy_revision_texts(self):
420
404
        source_vf, target_vf = self._build_vfs('revision', True, False)
426
410
 
427
411
    def _copy_inventory_texts(self):
428
412
        source_vf, target_vf = self._build_vfs('inventory', True, True)
429
 
        # It is not sufficient to just use self.revision_keys, as stacked
430
 
        # repositories can have more inventories than they have revisions.
431
 
        # One alternative would be to do something with
432
 
        # get_parent_map(self.revision_keys), but that shouldn't be any faster
433
 
        # than this.
434
 
        inventory_keys = source_vf.keys()
435
 
        missing_inventories = set(
436
 
            self.revision_keys).difference(inventory_keys)
437
 
        if missing_inventories:
438
 
            # Go back to the original repo, to see if these are really missing
439
 
            # https://bugs.launchpad.net/bzr/+bug/437003
440
 
            # If we are packing a subset of the repo, it is fine to just have
441
 
            # the data in another Pack file, which is not included in this pack
442
 
            # operation.
443
 
            inv_index = self._pack_collection.repo.inventories._index
444
 
            pmap = inv_index.get_parent_map(missing_inventories)
445
 
            really_missing = missing_inventories.difference(pmap)
446
 
            if really_missing:
447
 
                missing_inventories = sorted(really_missing)
448
 
                raise ValueError('We are missing inventories for revisions: %s'
449
 
                                 % (missing_inventories,))
450
 
        self._copy_stream(source_vf, target_vf, inventory_keys,
 
413
        self._copy_stream(source_vf, target_vf, self.revision_keys,
451
414
                          'inventories', self._get_filtered_inv_stream, 2)
452
415
 
453
 
    def _get_chk_vfs_for_copy(self):
454
 
        return self._build_vfs('chk', False, False)
455
 
 
456
416
    def _copy_chk_texts(self):
457
 
        source_vf, target_vf = self._get_chk_vfs_for_copy()
 
417
        source_vf, target_vf = self._build_vfs('chk', False, False)
458
418
        # TODO: This is technically spurious... if it is a performance issue,
459
419
        #       remove it
460
420
        total_keys = source_vf.keys()
463
423
                     len(self._chk_id_roots), len(self._chk_p_id_roots),
464
424
                     len(total_keys))
465
425
        self.pb.update('repacking chk', 3)
466
 
        with ui.ui_factory.nested_progress_bar() as child_pb:
 
426
        child_pb = ui.ui_factory.nested_progress_bar()
 
427
        try:
467
428
            for stream in self._get_chk_streams(source_vf, total_keys,
468
429
                                                pb=child_pb):
469
 
                for _, _ in target_vf._insert_record_stream(
470
 
                        stream, random_id=True, reuse_blocks=False):
 
430
                for _ in target_vf._insert_record_stream(stream,
 
431
                                                         random_id=True,
 
432
                                                         reuse_blocks=False):
471
433
                    pass
 
434
        finally:
 
435
            child_pb.finished()
472
436
 
473
437
    def _copy_text_texts(self):
474
438
        source_vf, target_vf = self._build_vfs('text', True, True)
492
456
        self.pb.update('repacking', 0, 7)
493
457
        self.new_pack = self.open_pack()
494
458
        # Is this necessary for GC ?
495
 
        self.new_pack.set_write_cache_size(1024 * 1024)
 
459
        self.new_pack.set_write_cache_size(1024*1024)
496
460
        self._copy_revision_texts()
497
461
        self._copy_inventory_texts()
498
462
        self._copy_chk_texts()
502
466
        if not self._use_pack(self.new_pack):
503
467
            self.new_pack.abort()
504
468
            return None
505
 
        self.new_pack.finish_content()
506
 
        if len(self.packs) == 1:
507
 
            old_pack = self.packs[0]
508
 
            if old_pack.name == self.new_pack._hash.hexdigest():
509
 
                # The single old pack was already optimally packed.
510
 
                trace.mutter('single pack %s was already optimally packed',
511
 
                             old_pack.name)
512
 
                self.new_pack.abort()
513
 
                return None
514
469
        self.pb.update('finishing repack', 6, 7)
515
470
        self.new_pack.finish()
516
471
        self._pack_collection.allocate(self.new_pack)
520
475
class GCCHKReconcilePacker(GCCHKPacker):
521
476
    """A packer which regenerates indices etc as it copies.
522
477
 
523
 
    This is used by ``brz reconcile`` to cause parent text pointers to be
 
478
    This is used by ``bzr reconcile`` to cause parent text pointers to be
524
479
    regenerated.
525
480
    """
526
481
 
549
504
        ancestor_keys = revision_vf.get_parent_map(revision_vf.keys())
550
505
        # Strip keys back into revision_ids.
551
506
        ancestors = dict((k[0], tuple([p[0] for p in parents]))
552
 
                         for k, parents in viewitems(ancestor_keys))
 
507
                         for k, parents in ancestor_keys.iteritems())
553
508
        del ancestor_keys
554
509
        # TODO: _generate_text_key_index should be much cheaper to generate from
555
510
        #       a chk repository, rather than the current implementation
558
513
        # 2) generate a keys list that contains all the entries that can
559
514
        #    be used as-is, with corrected parents.
560
515
        ok_keys = []
561
 
        new_parent_keys = {}  # (key, parent_keys)
 
516
        new_parent_keys = {} # (key, parent_keys)
562
517
        discarded_keys = []
563
518
        NULL_REVISION = _mod_revision.NULL_REVISION
564
519
        for key in self._text_refs:
588
543
        del ideal_index
589
544
        del file_id_parent_map
590
545
        # 3) bulk copy the data, updating records than need it
591
 
 
592
546
        def _update_parents_for_texts():
593
547
            stream = source_vf.get_record_stream(self._text_refs,
594
 
                                                 'groupcompress', False)
 
548
                'groupcompress', False)
595
549
            for record in stream:
596
550
                if record.key in new_parent_keys:
597
551
                    record.parents = new_parent_keys[record.key]
603
557
        return new_pack.data_inserted() and self._data_changed
604
558
 
605
559
 
606
 
class GCCHKCanonicalizingPacker(GCCHKPacker):
607
 
    """A packer that ensures inventories have canonical-form CHK maps.
608
 
 
609
 
    Ideally this would be part of reconcile, but it's very slow and rarely
610
 
    needed.  (It repairs repositories affected by
611
 
    https://bugs.launchpad.net/bzr/+bug/522637).
612
 
    """
613
 
 
614
 
    def __init__(self, *args, **kwargs):
615
 
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
616
 
        self._data_changed = False
617
 
 
618
 
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
619
 
        """Create and exhaust a stream, but don't insert it.
620
 
 
621
 
        This is useful to get the side-effects of generating a stream.
622
 
        """
623
 
        self.pb.update('scanning %s' % (message,), pb_offset)
624
 
        with ui.ui_factory.nested_progress_bar() as child_pb:
625
 
            list(vf_to_stream(source_vf, keys, message, child_pb))
626
 
 
627
 
    def _copy_inventory_texts(self):
628
 
        source_vf, target_vf = self._build_vfs('inventory', True, True)
629
 
        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
630
 
        inventory_keys = source_vf.keys()
631
 
        # First, copy the existing CHKs on the assumption that most of them
632
 
        # will be correct.  This will save us from having to reinsert (and
633
 
        # recompress) these records later at the cost of perhaps preserving a
634
 
        # few unused CHKs.
635
 
        # (Iterate but don't insert _get_filtered_inv_stream to populate the
636
 
        # variables needed by GCCHKPacker._copy_chk_texts.)
637
 
        self._exhaust_stream(source_vf, inventory_keys, 'inventories',
638
 
                             self._get_filtered_inv_stream, 2)
639
 
        GCCHKPacker._copy_chk_texts(self)
640
 
        # Now copy and fix the inventories, and any regenerated CHKs.
641
 
 
642
 
        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
643
 
            return self._get_filtered_canonicalizing_inv_stream(
644
 
                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
645
 
        self._copy_stream(source_vf, target_vf, inventory_keys,
646
 
                          'inventories', chk_canonicalizing_inv_stream, 4)
647
 
 
648
 
    def _copy_chk_texts(self):
649
 
        # No-op; in this class this happens during _copy_inventory_texts.
650
 
        pass
651
 
 
652
 
    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
653
 
                                                pb=None, source_chk_vf=None, target_chk_vf=None):
654
 
        """Filter the texts of inventories, regenerating CHKs to make sure they
655
 
        are canonical.
656
 
        """
657
 
        total_keys = len(keys)
658
 
        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
659
 
 
660
 
        def _filtered_inv_stream():
661
 
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
662
 
            search_key_name = None
663
 
            for idx, record in enumerate(stream):
664
 
                # Inventories should always be with revisions; assume success.
665
 
                lines = record.get_bytes_as('lines')
666
 
                chk_inv = inventory.CHKInventory.deserialise(
667
 
                    source_chk_vf, lines, record.key)
668
 
                if pb is not None:
669
 
                    pb.update('inv', idx, total_keys)
670
 
                chk_inv.id_to_entry._ensure_root()
671
 
                if search_key_name is None:
672
 
                    # Find the name corresponding to the search_key_func
673
 
                    search_key_reg = chk_map.search_key_registry
674
 
                    for search_key_name, func in viewitems(search_key_reg):
675
 
                        if func == chk_inv.id_to_entry._search_key_func:
676
 
                            break
677
 
                canonical_inv = inventory.CHKInventory.from_inventory(
678
 
                    target_chk_vf, chk_inv,
679
 
                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
680
 
                    search_key_name=search_key_name)
681
 
                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
682
 
                    trace.mutter(
683
 
                        'Non-canonical CHK map for id_to_entry of inv: %s '
684
 
                        '(root is %s, should be %s)' % (chk_inv.revision_id,
685
 
                                                        chk_inv.id_to_entry.key()[
686
 
                                                            0],
687
 
                                                        canonical_inv.id_to_entry.key()[0]))
688
 
                    self._data_changed = True
689
 
                p_id_map = chk_inv.parent_id_basename_to_file_id
690
 
                p_id_map._ensure_root()
691
 
                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
692
 
                if p_id_map.key() != canon_p_id_map.key():
693
 
                    trace.mutter(
694
 
                        'Non-canonical CHK map for parent_id_to_basename of '
695
 
                        'inv: %s (root is %s, should be %s)'
696
 
                        % (chk_inv.revision_id, p_id_map.key()[0],
697
 
                           canon_p_id_map.key()[0]))
698
 
                    self._data_changed = True
699
 
                yield versionedfile.ChunkedContentFactory(
700
 
                    record.key, record.parents, record.sha1, canonical_inv.to_lines(),
701
 
                    chunks_are_lines=True)
702
 
            # We have finished processing all of the inventory records, we
703
 
            # don't need these sets anymore
704
 
        return _filtered_inv_stream()
705
 
 
706
 
    def _use_pack(self, new_pack):
707
 
        """Override _use_pack to check for reconcile having changed content."""
708
 
        return new_pack.data_inserted() and self._data_changed
709
 
 
710
 
 
711
560
class GCRepositoryPackCollection(RepositoryPackCollection):
712
561
 
713
562
    pack_factory = GCPack
714
563
    resumed_pack_factory = ResumedGCPack
715
 
    normal_packer_class = GCCHKPacker
716
 
    optimising_packer_class = GCCHKPacker
717
 
 
718
 
    def _check_new_inventories(self):
719
 
        """Detect missing inventories or chk root entries for the new revisions
720
 
        in this write group.
721
 
 
722
 
        :returns: list of strs, summarising any problems found.  If the list is
723
 
            empty no problems were found.
 
564
 
 
565
    def _execute_pack_operations(self, pack_operations,
 
566
                                 _packer_class=GCCHKPacker,
 
567
                                 reload_func=None):
 
568
        """Execute a series of pack operations.
 
569
 
 
570
        :param pack_operations: A list of [revision_count, packs_to_combine].
 
571
        :param _packer_class: The class of packer to use (default: Packer).
 
572
        :return: None.
724
573
        """
725
 
        # Ensure that all revisions added in this write group have:
726
 
        #   - corresponding inventories,
727
 
        #   - chk root entries for those inventories,
728
 
        #   - and any present parent inventories have their chk root
729
 
        #     entries too.
730
 
        # And all this should be independent of any fallback repository.
731
 
        problems = []
732
 
        key_deps = self.repo.revisions._index._key_dependencies
733
 
        new_revisions_keys = key_deps.get_new_keys()
734
 
        no_fallback_inv_index = self.repo.inventories._index
735
 
        no_fallback_chk_bytes_index = self.repo.chk_bytes._index
736
 
        no_fallback_texts_index = self.repo.texts._index
737
 
        inv_parent_map = no_fallback_inv_index.get_parent_map(
738
 
            new_revisions_keys)
739
 
        # Are any inventories for corresponding to the new revisions missing?
740
 
        corresponding_invs = set(inv_parent_map)
741
 
        missing_corresponding = set(new_revisions_keys)
742
 
        missing_corresponding.difference_update(corresponding_invs)
743
 
        if missing_corresponding:
744
 
            problems.append("inventories missing for revisions %s" %
745
 
                            (sorted(missing_corresponding),))
746
 
            return problems
747
 
        # Are any chk root entries missing for any inventories?  This includes
748
 
        # any present parent inventories, which may be used when calculating
749
 
        # deltas for streaming.
750
 
        all_inv_keys = set(corresponding_invs)
751
 
        for parent_inv_keys in viewvalues(inv_parent_map):
752
 
            all_inv_keys.update(parent_inv_keys)
753
 
        # Filter out ghost parents.
754
 
        all_inv_keys.intersection_update(
755
 
            no_fallback_inv_index.get_parent_map(all_inv_keys))
756
 
        parent_invs_only_keys = all_inv_keys.symmetric_difference(
757
 
            corresponding_invs)
758
 
        all_missing = set()
759
 
        inv_ids = [key[-1] for key in all_inv_keys]
760
 
        parent_invs_only_ids = [key[-1] for key in parent_invs_only_keys]
761
 
        root_key_info = _build_interesting_key_sets(
762
 
            self.repo, inv_ids, parent_invs_only_ids)
763
 
        expected_chk_roots = root_key_info.all_keys()
764
 
        present_chk_roots = no_fallback_chk_bytes_index.get_parent_map(
765
 
            expected_chk_roots)
766
 
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
767
 
        if missing_chk_roots:
768
 
            problems.append(
769
 
                "missing referenced chk root keys: %s."
770
 
                "Run 'brz reconcile --canonicalize-chks' on the affected "
771
 
                "repository."
772
 
                % (sorted(missing_chk_roots),))
773
 
            # Don't bother checking any further.
774
 
            return problems
775
 
        # Find all interesting chk_bytes records, and make sure they are
776
 
        # present, as well as the text keys they reference.
777
 
        chk_bytes_no_fallbacks = self.repo.chk_bytes.without_fallbacks()
778
 
        chk_bytes_no_fallbacks._search_key_func = \
779
 
            self.repo.chk_bytes._search_key_func
780
 
        chk_diff = chk_map.iter_interesting_nodes(
781
 
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
782
 
            root_key_info.uninteresting_root_keys)
783
 
        text_keys = set()
784
 
        try:
785
 
            for record in _filter_text_keys(chk_diff, text_keys,
786
 
                                            chk_map._bytes_to_text_key):
787
 
                pass
788
 
        except errors.NoSuchRevision as e:
789
 
            # XXX: It would be nice if we could give a more precise error here.
790
 
            problems.append("missing chk node(s) for id_to_entry maps")
791
 
        chk_diff = chk_map.iter_interesting_nodes(
792
 
            chk_bytes_no_fallbacks, root_key_info.interesting_pid_root_keys,
793
 
            root_key_info.uninteresting_pid_root_keys)
794
 
        try:
795
 
            for interesting_rec, interesting_map in chk_diff:
796
 
                pass
797
 
        except errors.NoSuchRevision as e:
798
 
            problems.append(
799
 
                "missing chk node(s) for parent_id_basename_to_file_id maps")
800
 
        present_text_keys = no_fallback_texts_index.get_parent_map(text_keys)
801
 
        missing_text_keys = text_keys.difference(present_text_keys)
802
 
        if missing_text_keys:
803
 
            problems.append("missing text keys: %r"
804
 
                            % (sorted(missing_text_keys),))
805
 
        return problems
806
 
 
807
 
 
808
 
class CHKInventoryRepository(PackRepository):
809
 
    """subclass of PackRepository that uses CHK based inventories."""
810
 
 
811
 
    def __init__(self, _format, a_controldir, control_files, _commit_builder_class,
812
 
                 _serializer):
 
574
        # XXX: Copied across from RepositoryPackCollection simply because we
 
575
        #      want to override the _packer_class ... :(
 
576
        for revision_count, packs in pack_operations:
 
577
            # we may have no-ops from the setup logic
 
578
            if len(packs) == 0:
 
579
                continue
 
580
            packer = GCCHKPacker(self, packs, '.autopack',
 
581
                                 reload_func=reload_func)
 
582
            try:
 
583
                packer.pack()
 
584
            except errors.RetryWithNewPacks:
 
585
                # An exception is propagating out of this context, make sure
 
586
                # this packer has cleaned up. Packer() doesn't set its new_pack
 
587
                # state into the RepositoryPackCollection object, so we only
 
588
                # have access to it directly here.
 
589
                if packer.new_pack is not None:
 
590
                    packer.new_pack.abort()
 
591
                raise
 
592
            for pack in packs:
 
593
                self._remove_pack_from_memory(pack)
 
594
        # record the newly available packs and stop advertising the old
 
595
        # packs
 
596
        self._save_pack_names(clear_obsolete_packs=True)
 
597
        # Move the old packs out of the way now they are no longer referenced.
 
598
        for revision_count, packs in pack_operations:
 
599
            self._obsolete_packs(packs)
 
600
 
 
601
 
 
602
class CHKInventoryRepository(KnitPackRepository):
 
603
    """subclass of KnitPackRepository that uses CHK based inventories."""
 
604
 
 
605
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
 
606
        _serializer):
813
607
        """Overridden to change pack collection class."""
814
 
        super(CHKInventoryRepository, self).__init__(_format, a_controldir,
815
 
                                                     control_files, _commit_builder_class, _serializer)
 
608
        KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
 
609
            _commit_builder_class, _serializer)
 
610
        # and now replace everything it did :)
816
611
        index_transport = self._transport.clone('indices')
817
612
        self._pack_collection = GCRepositoryPackCollection(self,
818
 
                                                           self._transport, index_transport,
819
 
                                                           self._transport.clone(
820
 
                                                               'upload'),
821
 
                                                           self._transport.clone(
822
 
                                                               'packs'),
823
 
                                                           _format.index_builder_class,
824
 
                                                           _format.index_class,
825
 
                                                           use_chk_index=self._format.supports_chks,
826
 
                                                           )
 
613
            self._transport, index_transport,
 
614
            self._transport.clone('upload'),
 
615
            self._transport.clone('packs'),
 
616
            _format.index_builder_class,
 
617
            _format.index_class,
 
618
            use_chk_index=self._format.supports_chks,
 
619
            )
827
620
        self.inventories = GroupCompressVersionedFiles(
828
621
            _GCGraphIndex(self._pack_collection.inventory_index.combined_index,
829
 
                          add_callback=self._pack_collection.inventory_index.add_callback,
830
 
                          parents=True, is_locked=self.is_locked,
831
 
                          inconsistency_fatal=False),
 
622
                add_callback=self._pack_collection.inventory_index.add_callback,
 
623
                parents=True, is_locked=self.is_locked,
 
624
                inconsistency_fatal=False),
832
625
            access=self._pack_collection.inventory_index.data_access)
833
626
        self.revisions = GroupCompressVersionedFiles(
834
627
            _GCGraphIndex(self._pack_collection.revision_index.combined_index,
835
 
                          add_callback=self._pack_collection.revision_index.add_callback,
836
 
                          parents=True, is_locked=self.is_locked,
837
 
                          track_external_parent_refs=True, track_new_keys=True),
 
628
                add_callback=self._pack_collection.revision_index.add_callback,
 
629
                parents=True, is_locked=self.is_locked,
 
630
                track_external_parent_refs=True),
838
631
            access=self._pack_collection.revision_index.data_access,
839
632
            delta=False)
840
633
        self.signatures = GroupCompressVersionedFiles(
841
634
            _GCGraphIndex(self._pack_collection.signature_index.combined_index,
842
 
                          add_callback=self._pack_collection.signature_index.add_callback,
843
 
                          parents=False, is_locked=self.is_locked,
844
 
                          inconsistency_fatal=False),
 
635
                add_callback=self._pack_collection.signature_index.add_callback,
 
636
                parents=False, is_locked=self.is_locked,
 
637
                inconsistency_fatal=False),
845
638
            access=self._pack_collection.signature_index.data_access,
846
639
            delta=False)
847
640
        self.texts = GroupCompressVersionedFiles(
848
641
            _GCGraphIndex(self._pack_collection.text_index.combined_index,
849
 
                          add_callback=self._pack_collection.text_index.add_callback,
850
 
                          parents=True, is_locked=self.is_locked,
851
 
                          inconsistency_fatal=False),
 
642
                add_callback=self._pack_collection.text_index.add_callback,
 
643
                parents=True, is_locked=self.is_locked,
 
644
                inconsistency_fatal=False),
852
645
            access=self._pack_collection.text_index.data_access)
853
646
        # No parents, individual CHK pages don't have specific ancestry
854
647
        self.chk_bytes = GroupCompressVersionedFiles(
855
648
            _GCGraphIndex(self._pack_collection.chk_index.combined_index,
856
 
                          add_callback=self._pack_collection.chk_index.add_callback,
857
 
                          parents=False, is_locked=self.is_locked,
858
 
                          inconsistency_fatal=False),
 
649
                add_callback=self._pack_collection.chk_index.add_callback,
 
650
                parents=False, is_locked=self.is_locked,
 
651
                inconsistency_fatal=False),
859
652
            access=self._pack_collection.chk_index.data_access)
860
653
        search_key_name = self._format._serializer.search_key_name
861
654
        search_key_func = chk_map.search_key_registry.get(search_key_name)
882
675
        # make inventory
883
676
        serializer = self._format._serializer
884
677
        result = inventory.CHKInventory.from_inventory(self.chk_bytes, inv,
885
 
                                                       maximum_size=serializer.maximum_size,
886
 
                                                       search_key_name=serializer.search_key_name)
 
678
            maximum_size=serializer.maximum_size,
 
679
            search_key_name=serializer.search_key_name)
887
680
        inv_lines = result.to_lines()
888
681
        return self._inventory_add_lines(revision_id, parents,
889
 
                                         inv_lines, check_content=False)
 
682
            inv_lines, check_content=False)
890
683
 
891
684
    def _create_inv_from_null(self, delta, revision_id):
892
685
        """This will mutate new_inv directly.
910
703
                                 ' no new_path %r' % (file_id,))
911
704
            if new_path == '':
912
705
                new_inv.root_id = file_id
913
 
                parent_id_basename_key = StaticTuple(b'', b'').intern()
 
706
                parent_id_basename_key = ('', '')
914
707
            else:
915
708
                utf8_entry_name = entry.name.encode('utf-8')
916
 
                parent_id_basename_key = StaticTuple(entry.parent_id,
917
 
                                                     utf8_entry_name).intern()
 
709
                parent_id_basename_key = (entry.parent_id, utf8_entry_name)
918
710
            new_value = entry_to_bytes(entry)
919
711
            # Populate Caches?
920
712
            # new_inv._path_to_fileid_cache[new_path] = file_id
921
 
            key = StaticTuple(file_id).intern()
922
 
            id_to_entry_dict[key] = new_value
 
713
            id_to_entry_dict[(file_id,)] = new_value
923
714
            parent_id_basename_dict[parent_id_basename_key] = file_id
924
715
 
925
716
        new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
926
 
                                     parent_id_basename_dict, maximum_size=serializer.maximum_size)
 
717
            parent_id_basename_dict, maximum_size=serializer.maximum_size)
927
718
        return new_inv
928
719
 
929
720
    def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
955
746
            raise AssertionError("%r not in write group" % (self,))
956
747
        _mod_revision.check_not_reserved_id(new_revision_id)
957
748
        basis_tree = None
958
 
        if basis_inv is None or not isinstance(basis_inv, inventory.CHKInventory):
 
749
        if basis_inv is None:
959
750
            if basis_revision_id == _mod_revision.NULL_REVISION:
960
751
                new_inv = self._create_inv_from_null(delta, new_revision_id)
961
 
                if new_inv.root_id is None:
962
 
                    raise errors.RootMissing()
963
752
                inv_lines = new_inv.to_lines()
964
753
                return self._inventory_add_lines(new_revision_id, parents,
965
 
                                                 inv_lines, check_content=False), new_inv
 
754
                    inv_lines, check_content=False), new_inv
966
755
            else:
967
756
                basis_tree = self.revision_tree(basis_revision_id)
968
757
                basis_tree.lock_read()
969
 
                basis_inv = basis_tree.root_inventory
 
758
                basis_inv = basis_tree.inventory
970
759
        try:
971
760
            result = basis_inv.create_by_apply_delta(delta, new_revision_id,
972
 
                                                     propagate_caches=propagate_caches)
 
761
                propagate_caches=propagate_caches)
973
762
            inv_lines = result.to_lines()
974
763
            return self._inventory_add_lines(new_revision_id, parents,
975
 
                                             inv_lines, check_content=False), result
 
764
                inv_lines, check_content=False), result
976
765
        finally:
977
766
            if basis_tree is not None:
978
767
                basis_tree.unlock()
979
768
 
980
 
    def _deserialise_inventory(self, revision_id, lines):
981
 
        return inventory.CHKInventory.deserialise(self.chk_bytes, lines,
982
 
                                                  (revision_id,))
983
 
 
984
 
    def _iter_inventories(self, revision_ids, ordering):
 
769
    def _iter_inventories(self, revision_ids):
985
770
        """Iterate over many inventory objects."""
986
 
        if ordering is None:
987
 
            ordering = 'unordered'
988
771
        keys = [(revision_id,) for revision_id in revision_ids]
989
 
        stream = self.inventories.get_record_stream(keys, ordering, True)
 
772
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
990
773
        texts = {}
991
774
        for record in stream:
992
775
            if record.storage_kind != 'absent':
993
 
                texts[record.key] = record.get_bytes_as('lines')
 
776
                texts[record.key] = record.get_bytes_as('fulltext')
994
777
            else:
995
 
                texts[record.key] = None
 
778
                raise errors.NoSuchRevision(self, record.key)
996
779
        for key in keys:
997
 
            lines = texts[key]
998
 
            if lines is None:
999
 
                yield (None, key[-1])
1000
 
            else:
1001
 
                yield (inventory.CHKInventory.deserialise(
1002
 
                    self.chk_bytes, lines, key), key[-1])
 
780
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
1003
781
 
1004
 
    def _get_inventory_xml(self, revision_id):
1005
 
        """Get serialized inventory as a string."""
1006
 
        # Without a native 'xml' inventory, this method doesn't make sense.
1007
 
        # However older working trees, and older bundles want it - so we supply
1008
 
        # it allowing _get_inventory_xml to work. Bundles currently use the
1009
 
        # serializer directly; this also isn't ideal, but there isn't an xml
1010
 
        # iteration interface offered at all for repositories.
1011
 
        return self._serializer.write_inventory_to_lines(
1012
 
            self.get_inventory(revision_id))
 
782
    def _iter_inventory_xmls(self, revision_ids):
 
783
        # Without a native 'xml' inventory, this method doesn't make sense, so
 
784
        # make it raise to trap naughty direct users.
 
785
        raise NotImplementedError(self._iter_inventory_xmls)
1013
786
 
1014
787
    def _find_present_inventory_keys(self, revision_keys):
1015
788
        parent_map = self.inventories.get_parent_map(revision_keys)
1029
802
        rich_root = self.supports_rich_root()
1030
803
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1031
804
        file_id_revisions = {}
1032
 
        with ui.ui_factory.nested_progress_bar() as pb:
 
805
        pb = ui.ui_factory.nested_progress_bar()
 
806
        try:
1033
807
            revision_keys = [(r,) for r in revision_ids]
1034
808
            parent_keys = self._find_parent_keys_of_revisions(revision_keys)
1035
809
            # TODO: instead of using _find_present_inventory_keys, change the
1037
811
            #       However, we only want to tolerate missing parent
1038
812
            #       inventories, not missing inventories for revision_ids
1039
813
            present_parent_inv_keys = self._find_present_inventory_keys(
1040
 
                parent_keys)
1041
 
            present_parent_inv_ids = {k[-1] for k in present_parent_inv_keys}
 
814
                                        parent_keys)
 
815
            present_parent_inv_ids = set(
 
816
                [k[-1] for k in present_parent_inv_keys])
 
817
            uninteresting_root_keys = set()
 
818
            interesting_root_keys = set()
1042
819
            inventories_to_read = set(revision_ids)
1043
820
            inventories_to_read.update(present_parent_inv_ids)
1044
 
            root_key_info = _build_interesting_key_sets(
1045
 
                self, inventories_to_read, present_parent_inv_ids)
1046
 
            interesting_root_keys = root_key_info.interesting_root_keys
1047
 
            uninteresting_root_keys = root_key_info.uninteresting_root_keys
 
821
            for inv in self.iter_inventories(inventories_to_read):
 
822
                entry_chk_root_key = inv.id_to_entry.key()
 
823
                if inv.revision_id in present_parent_inv_ids:
 
824
                    uninteresting_root_keys.add(entry_chk_root_key)
 
825
                else:
 
826
                    interesting_root_keys.add(entry_chk_root_key)
 
827
 
1048
828
            chk_bytes = self.chk_bytes
1049
829
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
1050
 
                                                                interesting_root_keys, uninteresting_root_keys,
1051
 
                                                                pb=pb):
 
830
                        interesting_root_keys, uninteresting_root_keys,
 
831
                        pb=pb):
1052
832
                for name, bytes in items:
1053
833
                    (name_utf8, file_id, revision_id) = bytes_to_info(bytes)
1054
 
                    # TODO: consider interning file_id, revision_id here, or
1055
 
                    #       pushing that intern() into bytes_to_info()
1056
 
                    # TODO: rich_root should always be True here, for all
1057
 
                    #       repositories that support chk_bytes
1058
834
                    if not rich_root and name_utf8 == '':
1059
835
                        continue
1060
836
                    try:
1061
837
                        file_id_revisions[file_id].add(revision_id)
1062
838
                    except KeyError:
1063
 
                        file_id_revisions[file_id] = {revision_id}
 
839
                        file_id_revisions[file_id] = set([revision_id])
 
840
        finally:
 
841
            pb.finished()
1064
842
        return file_id_revisions
1065
843
 
1066
844
    def find_text_key_references(self):
1078
856
        revision_keys = self.revisions.keys()
1079
857
        result = {}
1080
858
        rich_roots = self.supports_rich_root()
1081
 
        with ui.ui_factory.nested_progress_bar() as pb:
 
859
        pb = ui.ui_factory.nested_progress_bar()
 
860
        try:
1082
861
            all_revs = self.all_revision_ids()
1083
862
            total = len(all_revs)
1084
863
            for pos, inv in enumerate(self.iter_inventories(all_revs)):
1091
870
                    if entry.revision == inv.revision_id:
1092
871
                        result[key] = True
1093
872
            return result
1094
 
 
1095
 
    def reconcile_canonicalize_chks(self):
1096
 
        """Reconcile this repository to make sure all CHKs are in canonical
1097
 
        form.
1098
 
        """
1099
 
        from .reconcile import PackReconciler
1100
 
        with self.lock_write():
1101
 
            reconciler = PackReconciler(
1102
 
                self, thorough=True, canonicalize_chks=True)
1103
 
            return reconciler.reconcile()
 
873
        finally:
 
874
            pb.finished()
1104
875
 
1105
876
    def _reconcile_pack(self, collection, packs, extension, revs, pb):
1106
877
        packer = GCCHKReconcilePacker(collection, packs, extension)
1107
878
        return packer.pack(pb)
1108
879
 
1109
 
    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
1110
 
        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
1111
 
        return packer.pack(pb)
1112
 
 
1113
880
    def _get_source(self, to_format):
1114
881
        """Return a source for streaming from this repository."""
1115
 
        if self._format._serializer == to_format._serializer:
 
882
        if isinstance(to_format, remote.RemoteRepositoryFormat):
 
883
            # Can't just check attributes on to_format with the current code,
 
884
            # work around this:
 
885
            to_format._ensure_real()
 
886
            to_format = to_format._custom_format
 
887
        if to_format.__class__ is self._format.__class__:
1116
888
            # We must be exactly the same format, otherwise stuff like the chk
1117
 
            # page layout might be different.
1118
 
            # Actually, this test is just slightly looser than exact so that
1119
 
            # CHK2 <-> 2a transfers will work.
 
889
            # page layout might be different
1120
890
            return GroupCHKStreamSource(self, to_format)
1121
891
        return super(CHKInventoryRepository, self)._get_source(to_format)
1122
892
 
1123
 
    def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1124
 
        """Find revisions with different parent lists in the revision object
1125
 
        and in the index graph.
1126
 
 
1127
 
        :param revisions_iterator: None, or an iterator of (revid,
1128
 
            Revision-or-None). This iterator controls the revisions checked.
1129
 
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
1130
 
            parents-in-revision).
1131
 
        """
1132
 
        if not self.is_locked():
1133
 
            raise AssertionError()
1134
 
        vf = self.revisions
1135
 
        if revisions_iterator is None:
1136
 
            revisions_iterator = self.iter_revisions(self.all_revision_ids())
1137
 
        for revid, revision in revisions_iterator:
1138
 
            if revision is None:
1139
 
                pass
1140
 
            parent_map = vf.get_parent_map([(revid,)])
1141
 
            parents_according_to_index = tuple(parent[-1] for parent in
1142
 
                                               parent_map[(revid,)])
1143
 
            parents_according_to_revision = tuple(revision.parent_ids)
1144
 
            if parents_according_to_index != parents_according_to_revision:
1145
 
                yield (revid, parents_according_to_index,
1146
 
                       parents_according_to_revision)
1147
 
 
1148
 
    def _check_for_inconsistent_revision_parents(self):
1149
 
        inconsistencies = list(self._find_inconsistent_revision_parents())
1150
 
        if inconsistencies:
1151
 
            raise errors.BzrCheckError(
1152
 
                "Revision index has inconsistent parents.")
1153
 
 
1154
 
 
1155
 
class GroupCHKStreamSource(StreamSource):
 
893
 
 
894
class GroupCHKStreamSource(KnitPackStreamSource):
1156
895
    """Used when both the source and target repo are GroupCHK repos."""
1157
896
 
1158
897
    def __init__(self, from_repository, to_format):
1172
911
        """
1173
912
        self._chk_id_roots = []
1174
913
        self._chk_p_id_roots = []
1175
 
 
1176
914
        def _filtered_inv_stream():
1177
915
            id_roots_set = set()
1178
916
            p_id_roots_set = set()
1185
923
                        continue
1186
924
                    else:
1187
925
                        raise errors.NoSuchRevision(self, record.key)
1188
 
                lines = record.get_bytes_as('lines')
1189
 
                chk_inv = inventory.CHKInventory.deserialise(None, lines,
 
926
                bytes = record.get_bytes_as('fulltext')
 
927
                chk_inv = inventory.CHKInventory.deserialise(None, bytes,
1190
928
                                                             record.key)
1191
929
                key = chk_inv.id_to_entry.key()
1192
930
                if key not in id_roots_set:
1218
956
            # TODO: Update Repository.iter_inventories() to add
1219
957
            #       ignore_missing=True
1220
958
            present_keys = self.from_repository._find_present_inventory_keys(
1221
 
                excluded_revision_keys)
 
959
                            excluded_revision_keys)
1222
960
            present_ids = [k[-1] for k in present_keys]
1223
961
            uninteresting_root_keys = set()
1224
962
            uninteresting_pid_root_keys = set()
1226
964
                uninteresting_root_keys.add(inv.id_to_entry.key())
1227
965
                uninteresting_pid_root_keys.add(
1228
966
                    inv.parent_id_basename_to_file_id.key())
 
967
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1229
968
        chk_bytes = self.from_repository.chk_bytes
1230
 
 
1231
969
        def _filter_id_to_entry():
1232
 
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
1233
 
                                                               self._chk_id_roots, uninteresting_root_keys)
1234
 
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
1235
 
                                            chk_map._bytes_to_text_key):
 
970
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
 
971
                        self._chk_id_roots, uninteresting_root_keys):
 
972
                for name, bytes in items:
 
973
                    # Note: we don't care about name_utf8, because we are always
 
974
                    # rich-root = True
 
975
                    _, file_id, revision_id = bytes_to_info(bytes)
 
976
                    self._text_keys.add((file_id, revision_id))
1236
977
                if record is not None:
1237
978
                    yield record
1238
979
            # Consumed
1239
980
            self._chk_id_roots = None
1240
981
        yield 'chk_bytes', _filter_id_to_entry()
1241
 
 
1242
982
        def _get_parent_id_basename_to_file_id_pages():
1243
983
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
1244
 
                                                                self._chk_p_id_roots, uninteresting_pid_root_keys):
 
984
                        self._chk_p_id_roots, uninteresting_pid_root_keys):
1245
985
                if record is not None:
1246
986
                    yield record
1247
987
            # Consumed
1248
988
            self._chk_p_id_roots = None
1249
989
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1250
990
 
1251
 
    def _get_text_stream(self):
1252
 
        # Note: We know we don't have to handle adding root keys, because both
1253
 
        # the source and target are the identical network name.
1254
 
        text_stream = self.from_repository.texts.get_record_stream(
1255
 
            self._text_keys, self._text_fetch_order, False)
1256
 
        return ('texts', text_stream)
1257
 
 
1258
991
    def get_stream(self, search):
1259
 
        def wrap_and_count(pb, rc, stream):
1260
 
            """Yield records from stream while showing progress."""
1261
 
            count = 0
1262
 
            for record in stream:
1263
 
                if count == rc.STEP:
1264
 
                    rc.increment(count)
1265
 
                    pb.update('Estimate', rc.current, rc.max)
1266
 
                    count = 0
1267
 
                count += 1
1268
 
                yield record
1269
 
 
1270
992
        revision_ids = search.get_keys()
1271
 
        with ui.ui_factory.nested_progress_bar() as pb:
1272
 
            rc = self._record_counter
1273
 
            self._record_counter.setup(len(revision_ids))
1274
 
            for stream_info in self._fetch_revision_texts(revision_ids):
1275
 
                yield (stream_info[0],
1276
 
                       wrap_and_count(pb, rc, stream_info[1]))
1277
 
            self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1278
 
            # TODO: The keys to exclude might be part of the search recipe
1279
 
            # For now, exclude all parents that are at the edge of ancestry, for
1280
 
            # which we have inventories
1281
 
            from_repo = self.from_repository
1282
 
            parent_keys = from_repo._find_parent_keys_of_revisions(
1283
 
                self._revision_keys)
1284
 
            self.from_repository.revisions.clear_cache()
1285
 
            self.from_repository.signatures.clear_cache()
1286
 
            # Clear the repo's get_parent_map cache too.
1287
 
            self.from_repository._unstacked_provider.disable_cache()
1288
 
            self.from_repository._unstacked_provider.enable_cache()
1289
 
            s = self._get_inventory_stream(self._revision_keys)
1290
 
            yield (s[0], wrap_and_count(pb, rc, s[1]))
1291
 
            self.from_repository.inventories.clear_cache()
1292
 
            for stream_info in self._get_filtered_chk_streams(parent_keys):
1293
 
                yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1294
 
            self.from_repository.chk_bytes.clear_cache()
1295
 
            s = self._get_text_stream()
1296
 
            yield (s[0], wrap_and_count(pb, rc, s[1]))
1297
 
            self.from_repository.texts.clear_cache()
1298
 
            pb.update('Done', rc.max, rc.max)
 
993
        for stream_info in self._fetch_revision_texts(revision_ids):
 
994
            yield stream_info
 
995
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
 
996
        yield self._get_inventory_stream(self._revision_keys)
 
997
        # TODO: The keys to exclude might be part of the search recipe
 
998
        # For now, exclude all parents that are at the edge of ancestry, for
 
999
        # which we have inventories
 
1000
        from_repo = self.from_repository
 
1001
        parent_keys = from_repo._find_parent_keys_of_revisions(
 
1002
                        self._revision_keys)
 
1003
        for stream_info in self._get_filtered_chk_streams(parent_keys):
 
1004
            yield stream_info
 
1005
        yield self._get_text_stream()
1299
1006
 
1300
1007
    def get_stream_for_missing_keys(self, missing_keys):
1301
1008
        # missing keys can only occur when we are byte copying and not
1305
1012
        for key in missing_keys:
1306
1013
            if key[0] != 'inventories':
1307
1014
                raise AssertionError('The only missing keys we should'
1308
 
                                     ' be filling in are inventory keys, not %s'
1309
 
                                     % (key[0],))
 
1015
                    ' be filling in are inventory keys, not %s'
 
1016
                    % (key[0],))
1310
1017
            missing_inventory_keys.add(key[1:])
1311
1018
        if self._chk_id_roots or self._chk_p_id_roots:
1312
1019
            raise AssertionError('Cannot call get_stream_for_missing_keys'
1313
 
                                 ' until all of get_stream() has been consumed.')
 
1020
                ' untill all of get_stream() has been consumed.')
1314
1021
        # Yield the inventory stream, so we can find the chk stream
1315
1022
        # Some of the missing_keys will be missing because they are ghosts.
1316
1023
        # As such, we can ignore them. The Sink is required to verify there are
1323
1030
            yield stream_info
1324
1031
 
1325
1032
 
1326
 
class _InterestingKeyInfo(object):
1327
 
    def __init__(self):
1328
 
        self.interesting_root_keys = set()
1329
 
        self.interesting_pid_root_keys = set()
1330
 
        self.uninteresting_root_keys = set()
1331
 
        self.uninteresting_pid_root_keys = set()
1332
 
 
1333
 
    def all_interesting(self):
1334
 
        return self.interesting_root_keys.union(self.interesting_pid_root_keys)
1335
 
 
1336
 
    def all_uninteresting(self):
1337
 
        return self.uninteresting_root_keys.union(
1338
 
            self.uninteresting_pid_root_keys)
1339
 
 
1340
 
    def all_keys(self):
1341
 
        return self.all_interesting().union(self.all_uninteresting())
1342
 
 
1343
 
 
1344
 
def _build_interesting_key_sets(repo, inventory_ids, parent_only_inv_ids):
1345
 
    result = _InterestingKeyInfo()
1346
 
    for inv in repo.iter_inventories(inventory_ids, 'unordered'):
1347
 
        root_key = inv.id_to_entry.key()
1348
 
        pid_root_key = inv.parent_id_basename_to_file_id.key()
1349
 
        if inv.revision_id in parent_only_inv_ids:
1350
 
            result.uninteresting_root_keys.add(root_key)
1351
 
            result.uninteresting_pid_root_keys.add(pid_root_key)
1352
 
        else:
1353
 
            result.interesting_root_keys.add(root_key)
1354
 
            result.interesting_pid_root_keys.add(pid_root_key)
1355
 
    return result
1356
 
 
1357
 
 
1358
 
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1359
 
    """Iterate the result of iter_interesting_nodes, yielding the records
1360
 
    and adding to text_keys.
1361
 
    """
1362
 
    text_keys_update = text_keys.update
1363
 
    for record, items in interesting_nodes_iterable:
1364
 
        text_keys_update([bytes_to_text_key(b) for n, b in items])
1365
 
        yield record
1366
 
 
1367
 
 
1368
 
class RepositoryFormat2a(RepositoryFormatPack):
1369
 
    """A CHK repository that uses the bencode revision serializer."""
 
1033
class RepositoryFormatCHK1(RepositoryFormatPack):
 
1034
    """A hashed CHK+group compress pack repository."""
1370
1035
 
1371
1036
    repository_class = CHKInventoryRepository
1372
1037
    supports_external_lookups = True
1373
1038
    supports_chks = True
1374
 
    _commit_builder_class = PackCommitBuilder
 
1039
    # For right now, setting this to True gives us InterModel1And2 rather
 
1040
    # than InterDifferingSerializer
 
1041
    _commit_builder_class = PackRootCommitBuilder
1375
1042
    rich_root_data = True
1376
 
    _serializer = chk_serializer.chk_bencode_serializer
 
1043
    _serializer = chk_serializer.chk_serializer_255_bigpage
1377
1044
    _commit_inv_deltas = True
1378
1045
    # What index classes to use
1379
1046
    index_builder_class = BTreeBuilder
1385
1052
    # multiple in-a-row (and sharing strings). Topological is better
1386
1053
    # for remote, because we access less data.
1387
1054
    _fetch_order = 'unordered'
1388
 
    # essentially ignored by the groupcompress code.
1389
 
    _fetch_uses_deltas = False
 
1055
    _fetch_uses_deltas = False # essentially ignored by the groupcompress code.
1390
1056
    fast_deltas = True
1391
1057
    pack_compresses = True
1392
 
    supports_tree_reference = True
1393
1058
 
1394
1059
    def _get_matching_bzrdir(self):
1395
 
        return controldir.format_registry.make_controldir('2a')
 
1060
        return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1396
1061
 
1397
1062
    def _ignore_setting_bzrdir(self, format):
1398
1063
        pass
1399
1064
 
1400
 
    _matchingcontroldir = property(
1401
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
1065
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1402
1066
 
1403
 
    @classmethod
1404
 
    def get_format_string(cls):
1405
 
        return b'Bazaar repository format 2a (needs bzr 1.16 or later)\n'
 
1067
    def get_format_string(self):
 
1068
        """See RepositoryFormat.get_format_string()."""
 
1069
        return ('Bazaar development format - group compression and chk inventory'
 
1070
                ' (needs bzr.dev from 1.14)\n')
1406
1071
 
1407
1072
    def get_format_description(self):
1408
1073
        """See RepositoryFormat.get_format_description()."""
1409
 
        return ("Repository format 2a - rich roots, group compression"
1410
 
                " and chk inventories")
1411
 
 
1412
 
 
1413
 
class RepositoryFormat2aSubtree(RepositoryFormat2a):
1414
 
    """A 2a repository format that supports nested trees.
1415
 
 
 
1074
        return ("Development repository format - rich roots, group compression"
 
1075
            " and chk inventories")
 
1076
 
 
1077
    def check_conversion_target(self, target_format):
 
1078
        if not target_format.rich_root_data:
 
1079
            raise errors.BadConversionTarget(
 
1080
                'Does not support rich root data.', target_format)
 
1081
        if (self.supports_tree_reference and 
 
1082
            not getattr(target_format, 'supports_tree_reference', False)):
 
1083
            raise errors.BadConversionTarget(
 
1084
                'Does not support nested trees', target_format)
 
1085
 
 
1086
 
 
1087
 
 
1088
class RepositoryFormatCHK2(RepositoryFormatCHK1):
 
1089
    """A CHK repository that uses the bencode revision serializer."""
 
1090
 
 
1091
    _serializer = chk_serializer.chk_bencode_serializer
 
1092
 
 
1093
    def _get_matching_bzrdir(self):
 
1094
        return bzrdir.format_registry.make_bzrdir('development7-rich-root')
 
1095
 
 
1096
    def _ignore_setting_bzrdir(self, format):
 
1097
        pass
 
1098
 
 
1099
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1100
 
 
1101
    def get_format_string(self):
 
1102
        """See RepositoryFormat.get_format_string()."""
 
1103
        return ('Bazaar development format - chk repository with bencode '
 
1104
                'revision serialization (needs bzr.dev from 1.16)\n')
 
1105
 
 
1106
 
 
1107
class RepositoryFormat2a(RepositoryFormatCHK2):
 
1108
    """A CHK repository that uses the bencode revision serializer.
 
1109
    
 
1110
    This is the same as RepositoryFormatCHK2 but with a public name.
1416
1111
    """
1417
1112
 
 
1113
    _serializer = chk_serializer.chk_bencode_serializer
 
1114
 
1418
1115
    def _get_matching_bzrdir(self):
1419
 
        return controldir.format_registry.make_controldir('development-subtree')
 
1116
        return bzrdir.format_registry.make_bzrdir('2a')
1420
1117
 
1421
1118
    def _ignore_setting_bzrdir(self, format):
1422
1119
        pass
1423
1120
 
1424
 
    _matchingcontroldir = property(
1425
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
1426
 
 
1427
 
    @classmethod
1428
 
    def get_format_string(cls):
1429
 
        return b'Bazaar development format 8\n'
1430
 
 
1431
 
    def get_format_description(self):
1432
 
        """See RepositoryFormat.get_format_description()."""
1433
 
        return ("Development repository format 8 - nested trees, "
1434
 
                "group compression and chk inventories")
1435
 
 
1436
 
    experimental = True
1437
 
    supports_tree_reference = True
 
1121
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1122
 
 
1123
    def get_format_string(self):
 
1124
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')