1
# Copyright (C) 2006-2010 Canonical Ltd
1
# Copyright (C) 2006-2011 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
19
19
from cStringIO import StringIO
33
from bzrlib.errors import (
34
RevisionAlreadyPresent,
32
from brzlib.errors import (
39
from bzrlib.index import *
40
from bzrlib.knit import (
36
from brzlib.index import *
37
from brzlib.knit import (
41
38
AnnotatedKnitContent,
44
40
KnitVersionedFiles,
46
42
_VFContentMapGenerator,
53
from bzrlib.repofmt import pack_repo
54
from bzrlib.tests import (
48
from brzlib.patiencediff import PatienceSequenceMatcher
49
from brzlib.repofmt import (
53
from brzlib.tests import (
58
55
TestCaseWithMemoryTransport,
59
56
TestCaseWithTransport,
62
from bzrlib.transport import get_transport
63
from bzrlib.transport.memory import MemoryTransport
64
from bzrlib.tuned_gzip import GzipFile
65
from bzrlib.versionedfile import (
59
from brzlib.versionedfile import (
66
60
AbsentContentFactory,
68
62
network_bytes_to_kind_and_offset,
69
63
RecordingVersionedFilesDecorator,
73
compiled_knit_feature = tests.ModuleAvailableFeature(
74
'bzrlib._knit_load_data_pyx')
65
from brzlib.tests import (
70
compiled_knit_feature = features.ModuleAvailableFeature(
71
'brzlib._knit_load_data_pyx')
77
74
class KnitContentTestsMixin(object):
106
103
line_delta = source_content.line_delta(target_content)
107
104
delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
108
105
source_lines, target_lines))
109
matcher = KnitSequenceMatcher(None, source_lines, target_lines)
110
matcher_blocks = list(list(matcher.get_matching_blocks()))
106
matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
107
matcher_blocks = list(matcher.get_matching_blocks())
111
108
self.assertEqual(matcher_blocks, delta_blocks)
113
110
def test_get_line_delta_blocks(self):
333
330
transport.append_bytes(packname, bytes)
334
331
writer = pack.ContainerWriter(write_data)
336
access = _DirectPackAccess({})
333
access = pack_repo._DirectPackAccess({})
337
334
access.set_writer(writer, index, (transport, packname))
338
335
return access, writer
346
def test_pack_collection_pack_retries(self):
347
"""An explicit pack of a pack collection succeeds even when a
348
concurrent pack happens.
350
builder = self.make_branch_builder('.')
351
builder.start_series()
352
builder.build_snapshot('rev-1', None, [
353
('add', ('', 'root-id', 'directory', None)),
354
('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
356
builder.build_snapshot('rev-2', ['rev-1'], [
357
('modify', ('file-id', 'content\nrev 2\n')),
359
builder.build_snapshot('rev-3', ['rev-2'], [
360
('modify', ('file-id', 'content\nrev 3\n')),
362
self.addCleanup(builder.finish_series)
363
b = builder.get_branch()
364
self.addCleanup(b.lock_write().unlock)
366
collection = repo._pack_collection
367
# Concurrently repack the repo.
368
reopened_repo = repo.bzrdir.open_repository()
349
373
def make_vf_for_retrying(self):
350
374
"""Create 3 packs and a reload function.
378
402
collection = repo._pack_collection
379
403
collection.ensure_loaded()
380
404
orig_packs = collection.packs
381
packer = pack_repo.Packer(collection, orig_packs, '.testpack')
405
packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
382
406
new_pack = packer.pack()
383
407
# forget about the new pack
384
408
collection.reset()
423
447
except _TestException, e:
424
448
retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
425
449
exc_info=sys.exc_info())
450
# GZ 2010-08-10: Cycle with exc_info affects 3 tests
428
453
def test_read_from_several_packs(self):
437
462
memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
439
464
transport = self.get_transport()
440
access = _DirectPackAccess({"FOO":(transport, 'packfile'),
465
access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
441
466
"FOOBAR":(transport, 'pack2'),
442
467
"BAZ":(transport, 'pack3')})
443
468
self.assertEqual(['1234567890', '12345', 'alpha'],
454
479
def test_set_writer(self):
455
480
"""The writer should be settable post construction."""
456
access = _DirectPackAccess({})
481
access = pack_repo._DirectPackAccess({})
457
482
transport = self.get_transport()
458
483
packname = 'packfile'
471
496
transport = self.get_transport()
472
497
reload_called, reload_func = self.make_reload_func()
473
498
# Note that the index key has changed from 'foo' to 'bar'
474
access = _DirectPackAccess({'bar':(transport, 'packname')},
499
access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
475
500
reload_func=reload_func)
476
501
e = self.assertListRaises(errors.RetryWithNewPacks,
477
502
access.get_raw_records, memos)
486
511
memos = self.make_pack_file()
487
512
transport = self.get_transport()
488
513
# Note that the index key has changed from 'foo' to 'bar'
489
access = _DirectPackAccess({'bar':(transport, 'packname')})
514
access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
490
515
e = self.assertListRaises(KeyError, access.get_raw_records, memos)
492
517
def test_missing_file_raises_retry(self):
494
519
transport = self.get_transport()
495
520
reload_called, reload_func = self.make_reload_func()
496
521
# Note that the 'filename' has been changed to 'different-packname'
497
access = _DirectPackAccess({'foo':(transport, 'different-packname')},
498
reload_func=reload_func)
522
access = pack_repo._DirectPackAccess(
523
{'foo':(transport, 'different-packname')},
524
reload_func=reload_func)
499
525
e = self.assertListRaises(errors.RetryWithNewPacks,
500
526
access.get_raw_records, memos)
501
527
# The file has gone missing, so we assume we need to reload
509
535
memos = self.make_pack_file()
510
536
transport = self.get_transport()
511
537
# Note that the 'filename' has been changed to 'different-packname'
512
access = _DirectPackAccess({'foo':(transport, 'different-packname')})
538
access = pack_repo._DirectPackAccess(
539
{'foo': (transport, 'different-packname')})
513
540
e = self.assertListRaises(errors.NoSuchFile,
514
541
access.get_raw_records, memos)
519
546
failing_transport = MockReadvFailingTransport(
520
547
[transport.get_bytes('packname')])
521
548
reload_called, reload_func = self.make_reload_func()
522
access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
523
reload_func=reload_func)
549
access = pack_repo._DirectPackAccess(
550
{'foo': (failing_transport, 'packname')},
551
reload_func=reload_func)
524
552
# Asking for a single record will not trigger the Mock failure
525
553
self.assertEqual(['1234567890'],
526
554
list(access.get_raw_records(memos[:1])))
542
570
failing_transport = MockReadvFailingTransport(
543
571
[transport.get_bytes('packname')])
544
572
reload_called, reload_func = self.make_reload_func()
545
access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
573
access = pack_repo._DirectPackAccess(
574
{'foo':(failing_transport, 'packname')})
546
575
# Asking for a single record will not trigger the Mock failure
547
576
self.assertEqual(['1234567890'],
548
577
list(access.get_raw_records(memos[:1])))
553
582
access.get_raw_records, memos)
555
584
def test_reload_or_raise_no_reload(self):
556
access = _DirectPackAccess({}, reload_func=None)
585
access = pack_repo._DirectPackAccess({}, reload_func=None)
557
586
retry_exc = self.make_retry_exception()
558
587
# Without a reload_func, we will just re-raise the original exception
559
588
self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
561
590
def test_reload_or_raise_reload_changed(self):
562
591
reload_called, reload_func = self.make_reload_func(return_val=True)
563
access = _DirectPackAccess({}, reload_func=reload_func)
592
access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
564
593
retry_exc = self.make_retry_exception()
565
594
access.reload_or_raise(retry_exc)
566
595
self.assertEqual([1], reload_called)
571
600
def test_reload_or_raise_reload_no_change(self):
572
601
reload_called, reload_func = self.make_reload_func(return_val=False)
573
access = _DirectPackAccess({}, reload_func=reload_func)
602
access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
574
603
retry_exc = self.make_retry_exception()
575
604
# If reload_occurred is False, then we consider it an error to have
576
605
# reload_func() return False (no changes).
708
737
def make_multiple_records(self):
709
738
"""Create the content for multiple records."""
710
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
739
sha1sum = osutils.sha_string('foo\nbar\n')
712
741
gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
717
746
record_1 = (0, len(gz_txt), sha1sum)
718
747
total_txt.append(gz_txt)
719
sha1sum = osutils.sha('baz\n').hexdigest()
748
sha1sum = osutils.sha_string('baz\n')
720
749
gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
726
755
return total_txt, record_1, record_2
728
757
def test_valid_knit_data(self):
729
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
758
sha1sum = osutils.sha_string('foo\nbar\n')
730
759
gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
765
794
def test_not_enough_lines(self):
766
sha1sum = osutils.sha('foo\n').hexdigest()
795
sha1sum = osutils.sha_string('foo\n')
767
796
# record says 2 lines data says 1
768
797
gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
781
810
self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
783
812
def test_too_many_lines(self):
784
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
813
sha1sum = osutils.sha_string('foo\nbar\n')
785
814
# record says 1 lines data says 2
786
815
gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
800
829
self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
802
831
def test_mismatched_version_id(self):
803
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
832
sha1sum = osutils.sha_string('foo\nbar\n')
804
833
gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
819
848
knit._read_records_iter_raw(records))
821
850
def test_uncompressed_data(self):
822
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
851
sha1sum = osutils.sha_string('foo\nbar\n')
823
852
txt = ('version rev-id-1 2 %s\n'
839
868
knit._read_records_iter_raw(records))
841
870
def test_corrupted_data(self):
842
sha1sum = osutils.sha('foo\nbar\n').hexdigest()
871
sha1sum = osutils.sha_string('foo\nbar\n')
843
872
gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
863
892
def get_knit_index(self, transport, name, mode):
864
893
mapper = ConstantMapper(name)
865
from bzrlib._knit_load_data_py import _load_data_py
894
from brzlib._knit_load_data_py import _load_data_py
866
895
self.overrideAttr(knit, '_load_data', _load_data_py)
867
896
allow_writes = lambda: 'w' in mode
868
897
return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
1167
1196
self.assertRaises(errors.KnitCorrupt, index.keys)
1168
1197
except TypeError, e:
1169
1198
if (str(e) == ('exceptions must be strings, classes, or instances,'
1170
' not exceptions.IndexError')
1171
and sys.version_info[0:2] >= (2,5)):
1199
' not exceptions.IndexError')):
1172
1200
self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1173
1201
' raising new style exceptions with python'
1187
1215
self.assertRaises(errors.KnitCorrupt, index.keys)
1188
1216
except TypeError, e:
1189
1217
if (str(e) == ('exceptions must be strings, classes, or instances,'
1190
' not exceptions.ValueError')
1191
and sys.version_info[0:2] >= (2,5)):
1218
' not exceptions.ValueError')):
1192
1219
self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1193
1220
' raising new style exceptions with python'
1207
1234
self.assertRaises(errors.KnitCorrupt, index.keys)
1208
1235
except TypeError, e:
1209
1236
if (str(e) == ('exceptions must be strings, classes, or instances,'
1210
' not exceptions.ValueError')
1211
and sys.version_info[0:2] >= (2,5)):
1237
' not exceptions.ValueError')):
1212
1238
self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1213
1239
' raising new style exceptions with python'
1225
1251
self.assertRaises(errors.KnitCorrupt, index.keys)
1226
1252
except TypeError, e:
1227
1253
if (str(e) == ('exceptions must be strings, classes, or instances,'
1228
' not exceptions.ValueError')
1229
and sys.version_info[0:2] >= (2,5)):
1254
' not exceptions.ValueError')):
1230
1255
self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1231
1256
' raising new style exceptions with python'
1243
1268
self.assertRaises(errors.KnitCorrupt, index.keys)
1244
1269
except TypeError, e:
1245
1270
if (str(e) == ('exceptions must be strings, classes, or instances,'
1246
' not exceptions.ValueError')
1247
and sys.version_info[0:2] >= (2,5)):
1271
' not exceptions.ValueError')):
1248
1272
self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1249
1273
' raising new style exceptions with python'
1299
1323
def get_knit_index(self, transport, name, mode):
1300
1324
mapper = ConstantMapper(name)
1301
from bzrlib._knit_load_data_pyx import _load_data_c
1325
from brzlib._knit_load_data_pyx import _load_data_c
1302
1326
self.overrideAttr(knit, '_load_data', _load_data_c)
1303
1327
allow_writes = lambda: mode == 'w'
1304
1328
return _KndxIndex(transport, mapper, lambda:None,
1579
1603
# could leave an empty .kndx file, which bzr would later claim was a
1580
1604
# corrupted file since the header was not present. In reality, the file
1581
1605
# just wasn't created, so it should be ignored.
1582
t = get_transport('.')
1606
t = transport.get_transport_from_path('.')
1583
1607
t.put_bytes('test.kndx', '')
1585
1609
knit = self.make_test_knit()
1587
1611
def test_knit_index_checks_header(self):
1588
t = get_transport('.')
1612
t = transport.get_transport_from_path('.')
1589
1613
t.put_bytes('test.kndx', '# not really a knit header\n\n')
1590
1614
k = self.make_test_knit()
1591
1615
self.assertRaises(KnitHeaderError, k.keys)
2419
2443
key_basis = ('bar',)
2420
2444
key_missing = ('missing',)
2421
2445
test.add_lines(key, (), ['foo\n'])
2422
key_sha1sum = osutils.sha('foo\n').hexdigest()
2446
key_sha1sum = osutils.sha_string('foo\n')
2423
2447
sha1s = test.get_sha1s([key])
2424
2448
self.assertEqual({key: key_sha1sum}, sha1s)
2425
2449
self.assertEqual([], basis.calls)
2427
2451
# directly (rather than via text reconstruction) so that remote servers
2428
2452
# etc don't have to answer with full content.
2429
2453
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2430
basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2454
basis_sha1sum = osutils.sha_string('foo\nbar\n')
2431
2455
basis.calls = []
2432
2456
sha1s = test.get_sha1s([key, key_missing, key_basis])
2433
2457
self.assertEqual({key: key_sha1sum,
added
removed
brzlib/tests/test_knit.py
