/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/tests/test_groupcompress.py

  • Committer: Jelmer Vernooij
  • Date: 2017-09-01 07:15:43 UTC
  • mfrom: (6770.3.2 py3_test_cleanup)
  • Revision ID: jelmer@jelmer.uk-20170901071543-1t83321xkog9qrxh
Merge lp:~gz/brz/py3_test_cleanup

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/tests/test_groupcompress.py
1
 
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
 
1
# Copyright (C) 2008-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
18
18
 
19
19
import zlib
20
20
 
21
 
from bzrlib import (
22
 
    btree_index,
23
 
    groupcompress,
 
21
from .. import (
 
22
    config,
24
23
    errors,
25
 
    index as _mod_index,
26
24
    osutils,
27
25
    tests,
28
26
    trace,
 
27
    )
 
28
from ..bzr import (
 
29
    btree_index,
 
30
    groupcompress,
 
31
    knit,
 
32
    index as _mod_index,
29
33
    versionedfile,
30
34
    )
31
 
from bzrlib.osutils import sha_string
32
 
from bzrlib.tests.test__groupcompress import compiled_groupcompress_feature
33
 
 
34
 
 
35
 
def load_tests(standard_tests, module, loader):
36
 
    """Parameterize tests for all versions of groupcompress."""
37
 
    to_adapt, result = tests.split_suite_by_condition(
38
 
        standard_tests, tests.condition_isinstance(TestAllGroupCompressors))
 
35
from ..osutils import sha_string
 
36
from .test__groupcompress import compiled_groupcompress_feature
 
37
from .scenarios import load_tests_apply_scenarios
 
38
 
 
39
 
 
40
def group_compress_implementation_scenarios():
39
41
    scenarios = [
40
42
        ('python', {'compressor': groupcompress.PythonGroupCompressor}),
41
43
        ]
42
44
    if compiled_groupcompress_feature.available():
43
45
        scenarios.append(('C',
44
46
            {'compressor': groupcompress.PyrexGroupCompressor}))
45
 
    return tests.multiply_tests(to_adapt, scenarios, result)
 
47
    return scenarios
 
48
 
 
49
 
 
50
load_tests = load_tests_apply_scenarios
46
51
 
47
52
 
48
53
class TestGroupCompressor(tests.TestCase):
66
71
class TestAllGroupCompressors(TestGroupCompressor):
67
72
    """Tests for GroupCompressor"""
68
73
 
69
 
    compressor = None # Set by multiply_tests
 
74
    scenarios = group_compress_implementation_scenarios()
 
75
    compressor = None # Set by scenario
70
76
 
71
77
    def test_empty_delta(self):
72
78
        compressor = self.compressor()
301
307
        for key in sorted(key_to_text):
302
308
            compressor.compress(key, key_to_text[key], None)
303
309
        locs = dict((key, (start, end)) for key, (start, _, end, _)
304
 
                    in compressor.labels_deltas.iteritems())
 
310
                    in compressor.labels_deltas.items())
305
311
        block = compressor.flush()
306
312
        raw_bytes = block.to_bytes()
307
313
        # Go through from_bytes(to_bytes()) so that we start with a compressed
347
353
        self.assertEqual(z_content, block._z_content)
348
354
        self.assertEqual(content, block._content)
349
355
 
 
356
    def test_to_chunks(self):
 
357
        content_chunks = ['this is some content\n',
 
358
                          'this content will be compressed\n']
 
359
        content_len = sum(map(len, content_chunks))
 
360
        content = ''.join(content_chunks)
 
361
        gcb = groupcompress.GroupCompressBlock()
 
362
        gcb.set_chunked_content(content_chunks, content_len)
 
363
        total_len, block_chunks = gcb.to_chunks()
 
364
        block_bytes = ''.join(block_chunks)
 
365
        self.assertEqual(gcb._z_content_length, len(gcb._z_content))
 
366
        self.assertEqual(total_len, len(block_bytes))
 
367
        self.assertEqual(gcb._content_length, content_len)
 
368
        expected_header =('gcb1z\n' # group compress block v1 zlib
 
369
                          '%d\n' # Length of compressed content
 
370
                          '%d\n' # Length of uncompressed content
 
371
                         ) % (gcb._z_content_length, gcb._content_length)
 
372
        # The first chunk should be the header chunk. It is small, fixed size,
 
373
        # and there is no compelling reason to split it up
 
374
        self.assertEqual(expected_header, block_chunks[0])
 
375
        self.assertStartsWith(block_bytes, expected_header)
 
376
        remaining_bytes = block_bytes[len(expected_header):]
 
377
        raw_bytes = zlib.decompress(remaining_bytes)
 
378
        self.assertEqual(content, raw_bytes)
 
379
 
350
380
    def test_to_bytes(self):
351
381
        content = ('this is some content\n'
352
382
                   'this content will be compressed\n')
379
409
        # partial decompression to work with. Most auto-generated data
380
410
        # compresses a bit too well, we want a combination, so we combine a sha
381
411
        # hash with compressible data.
382
 
        for i in xrange(2048):
 
412
        for i in range(2048):
383
413
            next_content = '%d\nThis is a bit of duplicate text\n' % (i,)
384
414
            content_chunks.append(next_content)
385
415
            next_sha1 = osutils.sha_string(next_content)
389
419
        z_content = zlib.compress(content)
390
420
        self.assertEqual(57182, len(z_content))
391
421
        block = groupcompress.GroupCompressBlock()
392
 
        block._z_content = z_content
 
422
        block._z_content_chunks = (z_content,)
393
423
        block._z_content_length = len(z_content)
394
424
        block._compressor_name = 'zlib'
395
425
        block._content_length = 158634
424
454
        # partial decompression to work with. Most auto-generated data
425
455
        # compresses a bit too well, we want a combination, so we combine a sha
426
456
        # hash with compressible data.
427
 
        for i in xrange(2048):
 
457
        for i in range(2048):
428
458
            next_content = '%d\nThis is a bit of duplicate text\n' % (i,)
429
459
            content_chunks.append(next_content)
430
460
            next_sha1 = osutils.sha_string(next_content)
434
464
        z_content = zlib.compress(content)
435
465
        self.assertEqual(57182, len(z_content))
436
466
        block = groupcompress.GroupCompressBlock()
437
 
        block._z_content = z_content
 
467
        block._z_content_chunks = (z_content,)
438
468
        block._z_content_length = len(z_content)
439
469
        block._compressor_name = 'zlib'
440
470
        block._content_length = 158634
526
556
                    'as-requested', False)]
527
557
        self.assertEqual([('b',), ('a',), ('d',), ('c',)], keys)
528
558
 
 
559
    def test_get_record_stream_max_bytes_to_index_default(self):
 
560
        vf = self.make_test_vf(True, dir='source')
 
561
        vf.add_lines(('a',), (), ['lines\n'])
 
562
        vf.writer.end()
 
563
        record = next(vf.get_record_stream([('a',)], 'unordered', True))
 
564
        self.assertEqual(vf._DEFAULT_COMPRESSOR_SETTINGS,
 
565
                         record._manager._get_compressor_settings())
 
566
 
 
567
    def test_get_record_stream_accesses_compressor_settings(self):
 
568
        vf = self.make_test_vf(True, dir='source')
 
569
        vf.add_lines(('a',), (), ['lines\n'])
 
570
        vf.writer.end()
 
571
        vf._max_bytes_to_index = 1234
 
572
        record = next(vf.get_record_stream([('a',)], 'unordered', True))
 
573
        self.assertEqual(dict(max_bytes_to_index=1234),
 
574
                         record._manager._get_compressor_settings())
 
575
 
529
576
    def test_insert_record_stream_reuses_blocks(self):
530
577
        vf = self.make_test_vf(True, dir='source')
531
578
        def grouped_stream(revision_ids, first_parents=()):
726
773
                         warnings)
727
774
 
728
775
    def test_inconsistent_redundant_inserts_raises(self):
729
 
        e = self.assertRaises(errors.KnitCorrupt, self.do_inconsistent_inserts,
 
776
        e = self.assertRaises(knit.KnitCorrupt, self.do_inconsistent_inserts,
730
777
                              inconsistency_fatal=True)
731
778
        self.assertContainsRe(str(e), "Knit.* corrupt: inconsistent details"
732
779
                              " in add_records:"
744
791
        self.assertEqual(0, len(vf._group_cache))
745
792
 
746
793
 
 
794
class TestGroupCompressConfig(tests.TestCaseWithTransport):
 
795
 
 
796
    def make_test_vf(self):
 
797
        t = self.get_transport('.')
 
798
        t.ensure_base()
 
799
        factory = groupcompress.make_pack_factory(graph=True,
 
800
            delta=False, keylength=1, inconsistency_fatal=True)
 
801
        vf = factory(t)
 
802
        self.addCleanup(groupcompress.cleanup_pack_group, vf)
 
803
        return vf
 
804
 
 
805
    def test_max_bytes_to_index_default(self):
 
806
        vf = self.make_test_vf()
 
807
        gc = vf._make_group_compressor()
 
808
        self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
 
809
                         vf._max_bytes_to_index)
 
810
        if isinstance(gc, groupcompress.PyrexGroupCompressor):
 
811
            self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
 
812
                             gc._delta_index._max_bytes_to_index)
 
813
 
 
814
    def test_max_bytes_to_index_in_config(self):
 
815
        c = config.GlobalConfig()
 
816
        c.set_user_option('bzr.groupcompress.max_bytes_to_index', '10000')
 
817
        vf = self.make_test_vf()
 
818
        gc = vf._make_group_compressor()
 
819
        self.assertEqual(10000, vf._max_bytes_to_index)
 
820
        if isinstance(gc, groupcompress.PyrexGroupCompressor):
 
821
            self.assertEqual(10000, gc._delta_index._max_bytes_to_index)
 
822
 
 
823
    def test_max_bytes_to_index_bad_config(self):
 
824
        c = config.GlobalConfig()
 
825
        c.set_user_option('bzr.groupcompress.max_bytes_to_index', 'boogah')
 
826
        vf = self.make_test_vf()
 
827
        # TODO: This is triggering a warning, we might want to trap and make
 
828
        #       sure it is readable.
 
829
        gc = vf._make_group_compressor()
 
830
        self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
 
831
                         vf._max_bytes_to_index)
 
832
        if isinstance(gc, groupcompress.PyrexGroupCompressor):
 
833
            self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
 
834
                             gc._delta_index._max_bytes_to_index)
 
835
 
747
836
 
748
837
class StubGCVF(object):
749
838
    def __init__(self, canned_get_blocks=None):
875
964
        for key in sorted(key_to_text):
876
965
            compressor.compress(key, key_to_text[key], None)
877
966
        locs = dict((key, (start, end)) for key, (start, _, end, _)
878
 
                    in compressor.labels_deltas.iteritems())
 
967
                    in compressor.labels_deltas.items())
879
968
        block = compressor.flush()
880
969
        raw_bytes = block.to_bytes()
881
970
        return locs, groupcompress.GroupCompressBlock.from_bytes(raw_bytes)
1020
1109
            self.assertEqual(self._texts[record.key],
1021
1110
                             record.get_bytes_as('fulltext'))
1022
1111
 
 
1112
    def test_manager_default_compressor_settings(self):
 
1113
        locations, old_block = self.make_block(self._texts)
 
1114
        manager = groupcompress._LazyGroupContentManager(old_block)
 
1115
        gcvf = groupcompress.GroupCompressVersionedFiles
 
1116
        # It doesn't greedily evaluate _max_bytes_to_index
 
1117
        self.assertIs(None, manager._compressor_settings)
 
1118
        self.assertEqual(gcvf._DEFAULT_COMPRESSOR_SETTINGS,
 
1119
                         manager._get_compressor_settings())
 
1120
 
 
1121
    def test_manager_custom_compressor_settings(self):
 
1122
        locations, old_block = self.make_block(self._texts)
 
1123
        called = []
 
1124
        def compressor_settings():
 
1125
            called.append('called')
 
1126
            return (10,)
 
1127
        manager = groupcompress._LazyGroupContentManager(old_block,
 
1128
            get_compressor_settings=compressor_settings)
 
1129
        gcvf = groupcompress.GroupCompressVersionedFiles
 
1130
        # It doesn't greedily evaluate compressor_settings
 
1131
        self.assertIs(None, manager._compressor_settings)
 
1132
        self.assertEqual((10,), manager._get_compressor_settings())
 
1133
        self.assertEqual((10,), manager._get_compressor_settings())
 
1134
        self.assertEqual((10,), manager._compressor_settings)
 
1135
        # Only called 1 time
 
1136
        self.assertEqual(['called'], called)
 
1137
 
 
1138
    def test__rebuild_handles_compressor_settings(self):
 
1139
        if not isinstance(groupcompress.GroupCompressor,
 
1140
                          groupcompress.PyrexGroupCompressor):
 
1141
            raise tests.TestNotApplicable('pure-python compressor'
 
1142
                ' does not handle compressor_settings')
 
1143
        locations, old_block = self.make_block(self._texts)
 
1144
        manager = groupcompress._LazyGroupContentManager(old_block,
 
1145
            get_compressor_settings=lambda: dict(max_bytes_to_index=32))
 
1146
        gc = manager._make_group_compressor()
 
1147
        self.assertEqual(32, gc._delta_index._max_bytes_to_index)
 
1148
        self.add_key_to_manager(('key3',), locations, old_block, manager)
 
1149
        self.add_key_to_manager(('key4',), locations, old_block, manager)
 
1150
        action, last_byte, total_bytes = manager._check_rebuild_action()
 
1151
        self.assertEqual('rebuild', action)
 
1152
        manager._rebuild_block()
 
1153
        new_block = manager._block
 
1154
        self.assertIsNot(old_block, new_block)
 
1155
        # Because of the new max_bytes_to_index, we do a poor job of
 
1156
        # rebuilding. This is a side-effect of the change, but at least it does
 
1157
        # show the setting had an effect.
 
1158
        self.assertTrue(old_block._content_length < new_block._content_length)
 
1159
 
1023
1160
    def test_check_is_well_utilized_all_keys(self):
1024
1161
        block, manager = self.make_block_and_full_manager(self._texts)
1025
1162
        self.assertFalse(manager.check_is_well_utilized())
1066
1203
        # consumption
1067
1204
        self.add_key_to_manager(('key4',), locations, block, manager)
1068
1205
        self.assertTrue(manager.check_is_well_utilized())
 
1206
 
 
1207
 
 
1208
class Test_GCBuildDetails(tests.TestCase):
 
1209
 
 
1210
    def test_acts_like_tuple(self):
 
1211
        # _GCBuildDetails inlines some of the data that used to be spread out
 
1212
        # across a bunch of tuples
 
1213
        bd = groupcompress._GCBuildDetails((('parent1',), ('parent2',)),
 
1214
            ('INDEX', 10, 20, 0, 5))
 
1215
        self.assertEqual(4, len(bd))
 
1216
        self.assertEqual(('INDEX', 10, 20, 0, 5), bd[0])
 
1217
        self.assertEqual(None, bd[1]) # Compression Parent is always None
 
1218
        self.assertEqual((('parent1',), ('parent2',)), bd[2])
 
1219
        self.assertEqual(('group', None), bd[3]) # Record details
 
1220
 
 
1221
    def test__repr__(self):
 
1222
        bd = groupcompress._GCBuildDetails((('parent1',), ('parent2',)),
 
1223
            ('INDEX', 10, 20, 0, 5))
 
1224
        self.assertEqual("_GCBuildDetails(('INDEX', 10, 20, 0, 5),"
 
1225
                         " (('parent1',), ('parent2',)))",
 
1226
                         repr(bd))
 
1227