/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

(mbp) stop using failIf, failUnless, etc (Martin Pool)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
 
1
# Copyright (C) 2008-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
32
32
    revision as _mod_revision,
33
33
    trace,
34
34
    ui,
 
35
    versionedfile,
35
36
    )
36
37
from bzrlib.btree_index import (
37
38
    BTreeGraphIndex,
38
39
    BTreeBuilder,
39
40
    )
 
41
from bzrlib.decorators import needs_write_lock
40
42
from bzrlib.groupcompress import (
41
43
    _GCGraphIndex,
42
44
    GroupCompressVersionedFiles,
43
45
    )
44
46
from bzrlib.repofmt.pack_repo import (
 
47
    _DirectPackAccess,
45
48
    Pack,
46
49
    NewPack,
47
 
    KnitPackRepository,
48
 
    KnitPackStreamSource,
 
50
    PackRepository,
49
51
    PackRootCommitBuilder,
50
52
    RepositoryPackCollection,
51
53
    RepositoryFormatPack,
52
54
    ResumedPack,
53
55
    Packer,
54
56
    )
 
57
from bzrlib.repository import (
 
58
    StreamSource,
 
59
    )
55
60
from bzrlib.static_tuple import StaticTuple
56
61
 
57
62
 
262
267
        remaining_keys = set(keys)
263
268
        counter = [0]
264
269
        if self._gather_text_refs:
265
 
            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
266
270
            self._text_refs = set()
267
271
        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):
268
272
            cur_keys = root_keys
289
293
                    # Store is None, because we know we have a LeafNode, and we
290
294
                    # just want its entries
291
295
                    for file_id, bytes in node.iteritems(None):
292
 
                        name_utf8, file_id, revision_id = bytes_to_info(bytes)
293
 
                        self._text_refs.add((file_id, revision_id))
 
296
                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))
294
297
                def next_stream():
295
298
                    stream = source_vf.get_record_stream(cur_keys,
296
299
                                                         'as-requested', True)
352
355
        """Build a VersionedFiles instance on top of this group of packs."""
353
356
        index_name = index_name + '_index'
354
357
        index_to_pack = {}
355
 
        access = knit._DirectPackAccess(index_to_pack,
356
 
                                        reload_func=self._reload_func)
 
358
        access = _DirectPackAccess(index_to_pack,
 
359
                                   reload_func=self._reload_func)
357
360
        if for_write:
358
361
            # Use new_pack
359
362
            if self.new_pack is None:
421
424
        inventory_keys = source_vf.keys()
422
425
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
423
426
        if missing_inventories:
424
 
            missing_inventories = sorted(missing_inventories)
425
 
            raise ValueError('We are missing inventories for revisions: %s'
426
 
                % (missing_inventories,))
 
427
            # Go back to the original repo, to see if these are really missing
 
428
            # https://bugs.launchpad.net/bzr/+bug/437003
 
429
            # If we are packing a subset of the repo, it is fine to just have
 
430
            # the data in another Pack file, which is not included in this pack
 
431
            # operation.
 
432
            inv_index = self._pack_collection.repo.inventories._index
 
433
            pmap = inv_index.get_parent_map(missing_inventories)
 
434
            really_missing = missing_inventories.difference(pmap)
 
435
            if really_missing:
 
436
                missing_inventories = sorted(really_missing)
 
437
                raise ValueError('We are missing inventories for revisions: %s'
 
438
                    % (missing_inventories,))
427
439
        self._copy_stream(source_vf, target_vf, inventory_keys,
428
440
                          'inventories', self._get_filtered_inv_stream, 2)
429
441
 
 
442
    def _get_chk_vfs_for_copy(self):
 
443
        return self._build_vfs('chk', False, False)
 
444
 
430
445
    def _copy_chk_texts(self):
431
 
        source_vf, target_vf = self._build_vfs('chk', False, False)
 
446
        source_vf, target_vf = self._get_chk_vfs_for_copy()
432
447
        # TODO: This is technically spurious... if it is a performance issue,
433
448
        #       remove it
434
449
        total_keys = source_vf.keys()
580
595
        return new_pack.data_inserted() and self._data_changed
581
596
 
582
597
 
 
598
class GCCHKCanonicalizingPacker(GCCHKPacker):
 
599
    """A packer that ensures inventories have canonical-form CHK maps.
 
600
    
 
601
    Ideally this would be part of reconcile, but it's very slow and rarely
 
602
    needed.  (It repairs repositories affected by
 
603
    https://bugs.launchpad.net/bzr/+bug/522637).
 
604
    """
 
605
 
 
606
    def __init__(self, *args, **kwargs):
 
607
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
 
608
        self._data_changed = False
 
609
    
 
610
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
 
611
        """Create and exhaust a stream, but don't insert it.
 
612
        
 
613
        This is useful to get the side-effects of generating a stream.
 
614
        """
 
615
        self.pb.update('scanning %s' % (message,), pb_offset)
 
616
        child_pb = ui.ui_factory.nested_progress_bar()
 
617
        try:
 
618
            list(vf_to_stream(source_vf, keys, message, child_pb))
 
619
        finally:
 
620
            child_pb.finished()
 
621
 
 
622
    def _copy_inventory_texts(self):
 
623
        source_vf, target_vf = self._build_vfs('inventory', True, True)
 
624
        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
 
625
        inventory_keys = source_vf.keys()
 
626
        # First, copy the existing CHKs on the assumption that most of them
 
627
        # will be correct.  This will save us from having to reinsert (and
 
628
        # recompress) these records later at the cost of perhaps preserving a
 
629
        # few unused CHKs. 
 
630
        # (Iterate but don't insert _get_filtered_inv_stream to populate the
 
631
        # variables needed by GCCHKPacker._copy_chk_texts.)
 
632
        self._exhaust_stream(source_vf, inventory_keys, 'inventories',
 
633
                self._get_filtered_inv_stream, 2)
 
634
        GCCHKPacker._copy_chk_texts(self)
 
635
        # Now copy and fix the inventories, and any regenerated CHKs.
 
636
        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
 
637
            return self._get_filtered_canonicalizing_inv_stream(
 
638
                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
 
639
        self._copy_stream(source_vf, target_vf, inventory_keys,
 
640
                          'inventories', chk_canonicalizing_inv_stream, 4)
 
641
 
 
642
    def _copy_chk_texts(self):
 
643
        # No-op; in this class this happens during _copy_inventory_texts.
 
644
        pass
 
645
 
 
646
    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
 
647
            pb=None, source_chk_vf=None, target_chk_vf=None):
 
648
        """Filter the texts of inventories, regenerating CHKs to make sure they
 
649
        are canonical.
 
650
        """
 
651
        total_keys = len(keys)
 
652
        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
 
653
        def _filtered_inv_stream():
 
654
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
 
655
            search_key_name = None
 
656
            for idx, record in enumerate(stream):
 
657
                # Inventories should always be with revisions; assume success.
 
658
                bytes = record.get_bytes_as('fulltext')
 
659
                chk_inv = inventory.CHKInventory.deserialise(
 
660
                    source_chk_vf, bytes, record.key)
 
661
                if pb is not None:
 
662
                    pb.update('inv', idx, total_keys)
 
663
                chk_inv.id_to_entry._ensure_root()
 
664
                if search_key_name is None:
 
665
                    # Find the name corresponding to the search_key_func
 
666
                    search_key_reg = chk_map.search_key_registry
 
667
                    for search_key_name, func in search_key_reg.iteritems():
 
668
                        if func == chk_inv.id_to_entry._search_key_func:
 
669
                            break
 
670
                canonical_inv = inventory.CHKInventory.from_inventory(
 
671
                    target_chk_vf, chk_inv,
 
672
                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
 
673
                    search_key_name=search_key_name)
 
674
                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
 
675
                    trace.mutter(
 
676
                        'Non-canonical CHK map for id_to_entry of inv: %s '
 
677
                        '(root is %s, should be %s)' % (chk_inv.revision_id,
 
678
                        chk_inv.id_to_entry.key()[0],
 
679
                        canonical_inv.id_to_entry.key()[0]))
 
680
                    self._data_changed = True
 
681
                p_id_map = chk_inv.parent_id_basename_to_file_id
 
682
                p_id_map._ensure_root()
 
683
                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
 
684
                if p_id_map.key() != canon_p_id_map.key():
 
685
                    trace.mutter(
 
686
                        'Non-canonical CHK map for parent_id_to_basename of '
 
687
                        'inv: %s (root is %s, should be %s)'
 
688
                        % (chk_inv.revision_id, p_id_map.key()[0],
 
689
                           canon_p_id_map.key()[0]))
 
690
                    self._data_changed = True
 
691
                yield versionedfile.ChunkedContentFactory(record.key,
 
692
                        record.parents, record.sha1,
 
693
                        canonical_inv.to_lines())
 
694
            # We have finished processing all of the inventory records, we
 
695
            # don't need these sets anymore
 
696
        return _filtered_inv_stream()
 
697
 
 
698
    def _use_pack(self, new_pack):
 
699
        """Override _use_pack to check for reconcile having changed content."""
 
700
        return new_pack.data_inserted() and self._data_changed
 
701
 
 
702
 
583
703
class GCRepositoryPackCollection(RepositoryPackCollection):
584
704
 
585
705
    pack_factory = GCPack
647
767
        chk_diff = chk_map.iter_interesting_nodes(
648
768
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
649
769
            root_key_info.uninteresting_root_keys)
650
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
651
770
        text_keys = set()
652
771
        try:
653
 
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
 
772
            for record in _filter_text_keys(chk_diff, text_keys,
 
773
                                            chk_map._bytes_to_text_key):
654
774
                pass
655
775
        except errors.NoSuchRevision, e:
656
776
            # XXX: It would be nice if we could give a more precise error here.
712
832
        return result
713
833
 
714
834
 
715
 
class CHKInventoryRepository(KnitPackRepository):
716
 
    """subclass of KnitPackRepository that uses CHK based inventories."""
 
835
class CHKInventoryRepository(PackRepository):
 
836
    """subclass of PackRepository that uses CHK based inventories."""
717
837
 
718
838
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
719
839
        _serializer):
720
840
        """Overridden to change pack collection class."""
721
 
        KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
 
841
        super(CHKInventoryRepository, self).__init__(_format, a_bzrdir, control_files,
722
842
            _commit_builder_class, _serializer)
723
 
        # and now replace everything it did :)
724
843
        index_transport = self._transport.clone('indices')
725
844
        self._pack_collection = GCRepositoryPackCollection(self,
726
845
            self._transport, index_transport,
864
983
        if basis_inv is None:
865
984
            if basis_revision_id == _mod_revision.NULL_REVISION:
866
985
                new_inv = self._create_inv_from_null(delta, new_revision_id)
 
986
                if new_inv.root_id is None:
 
987
                    raise errors.RootMissing()
867
988
                inv_lines = new_inv.to_lines()
868
989
                return self._inventory_add_lines(new_revision_id, parents,
869
990
                    inv_lines, check_content=False), new_inv
999
1120
        finally:
1000
1121
            pb.finished()
1001
1122
 
 
1123
    @needs_write_lock
 
1124
    def reconcile_canonicalize_chks(self):
 
1125
        """Reconcile this repository to make sure all CHKs are in canonical
 
1126
        form.
 
1127
        """
 
1128
        from bzrlib.reconcile import PackReconciler
 
1129
        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)
 
1130
        reconciler.reconcile()
 
1131
        return reconciler
 
1132
 
1002
1133
    def _reconcile_pack(self, collection, packs, extension, revs, pb):
1003
1134
        packer = GCCHKReconcilePacker(collection, packs, extension)
1004
1135
        return packer.pack(pb)
1005
1136
 
 
1137
    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
 
1138
        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
 
1139
        return packer.pack(pb)
 
1140
 
1006
1141
    def _get_source(self, to_format):
1007
1142
        """Return a source for streaming from this repository."""
1008
1143
        if self._format._serializer == to_format._serializer:
1014
1149
        return super(CHKInventoryRepository, self)._get_source(to_format)
1015
1150
 
1016
1151
 
1017
 
class GroupCHKStreamSource(KnitPackStreamSource):
 
1152
class GroupCHKStreamSource(StreamSource):
1018
1153
    """Used when both the source and target repo are GroupCHK repos."""
1019
1154
 
1020
1155
    def __init__(self, from_repository, to_format):
1087
1222
                uninteresting_root_keys.add(inv.id_to_entry.key())
1088
1223
                uninteresting_pid_root_keys.add(
1089
1224
                    inv.parent_id_basename_to_file_id.key())
1090
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1091
1225
        chk_bytes = self.from_repository.chk_bytes
1092
1226
        def _filter_id_to_entry():
1093
1227
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
1094
1228
                        self._chk_id_roots, uninteresting_root_keys)
1095
1229
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
1096
 
                    bytes_to_info):
 
1230
                    chk_map._bytes_to_text_key):
1097
1231
                if record is not None:
1098
1232
                    yield record
1099
1233
            # Consumed
1108
1242
            self._chk_p_id_roots = None
1109
1243
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1110
1244
 
 
1245
    def _get_text_stream(self):
 
1246
        # Note: We know we don't have to handle adding root keys, because both
 
1247
        # the source and target are the identical network name.
 
1248
        text_stream = self.from_repository.texts.get_record_stream(
 
1249
                        self._text_keys, self._text_fetch_order, False)
 
1250
        return ('texts', text_stream)
 
1251
 
1111
1252
    def get_stream(self, search):
 
1253
        def wrap_and_count(pb, rc, stream):
 
1254
            """Yield records from stream while showing progress."""
 
1255
            count = 0
 
1256
            for record in stream:
 
1257
                if count == rc.STEP:
 
1258
                    rc.increment(count)
 
1259
                    pb.update('Estimate', rc.current, rc.max)
 
1260
                    count = 0
 
1261
                count += 1
 
1262
                yield record
 
1263
 
1112
1264
        revision_ids = search.get_keys()
 
1265
        pb = ui.ui_factory.nested_progress_bar()
 
1266
        rc = self._record_counter
 
1267
        self._record_counter.setup(len(revision_ids))
1113
1268
        for stream_info in self._fetch_revision_texts(revision_ids):
1114
 
            yield stream_info
 
1269
            yield (stream_info[0],
 
1270
                wrap_and_count(pb, rc, stream_info[1]))
1115
1271
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1116
1272
        self.from_repository.revisions.clear_cache()
1117
1273
        self.from_repository.signatures.clear_cache()
1118
 
        yield self._get_inventory_stream(self._revision_keys)
 
1274
        s = self._get_inventory_stream(self._revision_keys)
 
1275
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1119
1276
        self.from_repository.inventories.clear_cache()
1120
1277
        # TODO: The keys to exclude might be part of the search recipe
1121
1278
        # For now, exclude all parents that are at the edge of ancestry, for
1124
1281
        parent_keys = from_repo._find_parent_keys_of_revisions(
1125
1282
                        self._revision_keys)
1126
1283
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1127
 
            yield stream_info
 
1284
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1128
1285
        self.from_repository.chk_bytes.clear_cache()
1129
 
        yield self._get_text_stream()
 
1286
        s = self._get_text_stream()
 
1287
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1130
1288
        self.from_repository.texts.clear_cache()
 
1289
        pb.update('Done', rc.max, rc.max)
 
1290
        pb.finished()
1131
1291
 
1132
1292
    def get_stream_for_missing_keys(self, missing_keys):
1133
1293
        # missing keys can only occur when we are byte copying and not
1187
1347
    return result
1188
1348
 
1189
1349
 
1190
 
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
 
1350
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1191
1351
    """Iterate the result of iter_interesting_nodes, yielding the records
1192
1352
    and adding to text_keys.
1193
1353
    """
 
1354
    text_keys_update = text_keys.update
1194
1355
    for record, items in interesting_nodes_iterable:
1195
 
        for name, bytes in items:
1196
 
            # Note: we don't care about name_utf8, because groupcompress repos
1197
 
            # are always rich-root, so there are no synthesised root records to
1198
 
            # ignore.
1199
 
            _, file_id, revision_id = bytes_to_info(bytes)
1200
 
            file_id = intern(file_id)
1201
 
            revision_id = intern(revision_id)
1202
 
            text_keys.add(StaticTuple(file_id, revision_id).intern())
 
1356
        text_keys_update([bytes_to_text_key(b) for n,b in items])
1203
1357
        yield record
1204
1358
 
1205
1359
 
1206
 
 
1207
 
 
1208
 
class RepositoryFormatCHK1(RepositoryFormatPack):
1209
 
    """A hashed CHK+group compress pack repository."""
 
1360
class RepositoryFormat2a(RepositoryFormatPack):
 
1361
    """A CHK repository that uses the bencode revision serializer."""
1210
1362
 
1211
1363
    repository_class = CHKInventoryRepository
1212
1364
    supports_external_lookups = True
1213
1365
    supports_chks = True
1214
 
    # For right now, setting this to True gives us InterModel1And2 rather
1215
 
    # than InterDifferingSerializer
1216
1366
    _commit_builder_class = PackRootCommitBuilder
1217
1367
    rich_root_data = True
1218
 
    _serializer = chk_serializer.chk_serializer_255_bigpage
 
1368
    _serializer = chk_serializer.chk_bencode_serializer
1219
1369
    _commit_inv_deltas = True
1220
1370
    # What index classes to use
1221
1371
    index_builder_class = BTreeBuilder
1232
1382
    pack_compresses = True
1233
1383
 
1234
1384
    def _get_matching_bzrdir(self):
1235
 
        return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1236
 
 
1237
 
    def _ignore_setting_bzrdir(self, format):
1238
 
        pass
1239
 
 
1240
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1241
 
 
1242
 
    def get_format_string(self):
1243
 
        """See RepositoryFormat.get_format_string()."""
1244
 
        return ('Bazaar development format - group compression and chk inventory'
1245
 
                ' (needs bzr.dev from 1.14)\n')
1246
 
 
1247
 
    def get_format_description(self):
1248
 
        """See RepositoryFormat.get_format_description()."""
1249
 
        return ("Development repository format - rich roots, group compression"
1250
 
            " and chk inventories")
1251
 
 
1252
 
 
1253
 
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1254
 
    """A CHK repository that uses the bencode revision serializer."""
1255
 
 
1256
 
    _serializer = chk_serializer.chk_bencode_serializer
1257
 
 
1258
 
    def _get_matching_bzrdir(self):
1259
 
        return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1260
 
 
1261
 
    def _ignore_setting_bzrdir(self, format):
1262
 
        pass
1263
 
 
1264
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1265
 
 
1266
 
    def get_format_string(self):
1267
 
        """See RepositoryFormat.get_format_string()."""
1268
 
        return ('Bazaar development format - chk repository with bencode '
1269
 
                'revision serialization (needs bzr.dev from 1.16)\n')
1270
 
 
1271
 
 
1272
 
class RepositoryFormat2a(RepositoryFormatCHK2):
1273
 
    """A CHK repository that uses the bencode revision serializer.
1274
 
 
1275
 
    This is the same as RepositoryFormatCHK2 but with a public name.
1276
 
    """
1277
 
 
1278
 
    _serializer = chk_serializer.chk_bencode_serializer
1279
 
 
1280
 
    def _get_matching_bzrdir(self):
1281
1385
        return bzrdir.format_registry.make_bzrdir('2a')
1282
1386
 
1283
1387
    def _ignore_setting_bzrdir(self, format):
1292
1396
        """See RepositoryFormat.get_format_description()."""
1293
1397
        return ("Repository format 2a - rich roots, group compression"
1294
1398
            " and chk inventories")
 
1399
 
 
1400
 
 
1401
class RepositoryFormat2aSubtree(RepositoryFormat2a):
 
1402
    """A 2a repository format that supports nested trees.
 
1403
 
 
1404
    """
 
1405
 
 
1406
    def _get_matching_bzrdir(self):
 
1407
        return bzrdir.format_registry.make_bzrdir('development-subtree')
 
1408
 
 
1409
    def _ignore_setting_bzrdir(self, format):
 
1410
        pass
 
1411
 
 
1412
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1413
 
 
1414
    def get_format_string(self):
 
1415
        return ('Bazaar development format 8\n')
 
1416
 
 
1417
    def get_format_description(self):
 
1418
        """See RepositoryFormat.get_format_description()."""
 
1419
        return ("Development repository format 8 - nested trees, "
 
1420
                "group compression and chk inventories")
 
1421
 
 
1422
    experimental = True
 
1423
    supports_tree_reference = True