340
472
writer = pack.ContainerWriter(write_data)
342
474
access.set_writer(writer, index, (transport, packname))
343
memos = access.add_raw_records([10], '1234567890')
475
memos = access.add_raw_records([('key', 10)], '1234567890')
345
477
self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
479
def test_missing_index_raises_retry(self):
480
memos = self.make_pack_file()
481
transport = self.get_transport()
482
reload_called, reload_func = self.make_reload_func()
483
# Note that the index key has changed from 'foo' to 'bar'
484
access = _DirectPackAccess({'bar':(transport, 'packname')},
485
reload_func=reload_func)
486
e = self.assertListRaises(errors.RetryWithNewPacks,
487
access.get_raw_records, memos)
488
# Because a key was passed in which does not match our index list, we
489
# assume that the listing was already reloaded
490
self.assertTrue(e.reload_occurred)
491
self.assertIsInstance(e.exc_info, tuple)
492
self.assertIs(e.exc_info[0], KeyError)
493
self.assertIsInstance(e.exc_info[1], KeyError)
495
def test_missing_index_raises_key_error_with_no_reload(self):
496
memos = self.make_pack_file()
497
transport = self.get_transport()
498
# Note that the index key has changed from 'foo' to 'bar'
499
access = _DirectPackAccess({'bar':(transport, 'packname')})
500
e = self.assertListRaises(KeyError, access.get_raw_records, memos)
502
def test_missing_file_raises_retry(self):
503
memos = self.make_pack_file()
504
transport = self.get_transport()
505
reload_called, reload_func = self.make_reload_func()
506
# Note that the 'filename' has been changed to 'different-packname'
507
access = _DirectPackAccess({'foo':(transport, 'different-packname')},
508
reload_func=reload_func)
509
e = self.assertListRaises(errors.RetryWithNewPacks,
510
access.get_raw_records, memos)
511
# The file has gone missing, so we assume we need to reload
512
self.assertFalse(e.reload_occurred)
513
self.assertIsInstance(e.exc_info, tuple)
514
self.assertIs(e.exc_info[0], errors.NoSuchFile)
515
self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
516
self.assertEqual('different-packname', e.exc_info[1].path)
518
def test_missing_file_raises_no_such_file_with_no_reload(self):
519
memos = self.make_pack_file()
520
transport = self.get_transport()
521
# Note that the 'filename' has been changed to 'different-packname'
522
access = _DirectPackAccess({'foo':(transport, 'different-packname')})
523
e = self.assertListRaises(errors.NoSuchFile,
524
access.get_raw_records, memos)
526
def test_failing_readv_raises_retry(self):
527
memos = self.make_pack_file()
528
transport = self.get_transport()
529
failing_transport = MockReadvFailingTransport(
530
[transport.get_bytes('packname')])
531
reload_called, reload_func = self.make_reload_func()
532
access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
533
reload_func=reload_func)
534
# Asking for a single record will not trigger the Mock failure
535
self.assertEqual(['1234567890'],
536
list(access.get_raw_records(memos[:1])))
537
self.assertEqual(['12345'],
538
list(access.get_raw_records(memos[1:2])))
539
# A multiple offset readv() will fail mid-way through
540
e = self.assertListRaises(errors.RetryWithNewPacks,
541
access.get_raw_records, memos)
542
# The file has gone missing, so we assume we need to reload
543
self.assertFalse(e.reload_occurred)
544
self.assertIsInstance(e.exc_info, tuple)
545
self.assertIs(e.exc_info[0], errors.NoSuchFile)
546
self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
547
self.assertEqual('packname', e.exc_info[1].path)
549
def test_failing_readv_raises_no_such_file_with_no_reload(self):
550
memos = self.make_pack_file()
551
transport = self.get_transport()
552
failing_transport = MockReadvFailingTransport(
553
[transport.get_bytes('packname')])
554
reload_called, reload_func = self.make_reload_func()
555
access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
556
# Asking for a single record will not trigger the Mock failure
557
self.assertEqual(['1234567890'],
558
list(access.get_raw_records(memos[:1])))
559
self.assertEqual(['12345'],
560
list(access.get_raw_records(memos[1:2])))
561
# A multiple offset readv() will fail mid-way through
562
e = self.assertListRaises(errors.NoSuchFile,
563
access.get_raw_records, memos)
565
def test_reload_or_raise_no_reload(self):
566
access = _DirectPackAccess({}, reload_func=None)
567
retry_exc = self.make_retry_exception()
568
# Without a reload_func, we will just re-raise the original exception
569
self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
571
def test_reload_or_raise_reload_changed(self):
572
reload_called, reload_func = self.make_reload_func(return_val=True)
573
access = _DirectPackAccess({}, reload_func=reload_func)
574
retry_exc = self.make_retry_exception()
575
access.reload_or_raise(retry_exc)
576
self.assertEqual([1], reload_called)
577
retry_exc.reload_occurred=True
578
access.reload_or_raise(retry_exc)
579
self.assertEqual([2], reload_called)
581
def test_reload_or_raise_reload_no_change(self):
582
reload_called, reload_func = self.make_reload_func(return_val=False)
583
access = _DirectPackAccess({}, reload_func=reload_func)
584
retry_exc = self.make_retry_exception()
585
# If reload_occurred is False, then we consider it an error to have
586
# reload_func() return False (no changes).
587
self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
588
self.assertEqual([1], reload_called)
589
retry_exc.reload_occurred=True
590
# If reload_occurred is True, then we assume nothing changed because
591
# it had changed earlier, but didn't change again
592
access.reload_or_raise(retry_exc)
593
self.assertEqual([2], reload_called)
595
def test_annotate_retries(self):
596
vf, reload_counter = self.make_vf_for_retrying()
597
# It is a little bit bogus to annotate the Revision VF, but it works,
598
# as we have ancestry stored there
600
reload_lines = vf.annotate(key)
601
self.assertEqual([1, 1, 0], reload_counter)
602
plain_lines = vf.annotate(key)
603
self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
604
if reload_lines != plain_lines:
605
self.fail('Annotation was not identical with reloading.')
606
# Now delete the packs-in-use, which should trigger another reload, but
607
# this time we just raise an exception because we can't recover
608
for trans, name in vf._access._indices.itervalues():
610
self.assertRaises(errors.NoSuchFile, vf.annotate, key)
611
self.assertEqual([2, 1, 1], reload_counter)
613
def test__get_record_map_retries(self):
614
vf, reload_counter = self.make_vf_for_retrying()
615
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
616
records = vf._get_record_map(keys)
617
self.assertEqual(keys, sorted(records.keys()))
618
self.assertEqual([1, 1, 0], reload_counter)
619
# Now delete the packs-in-use, which should trigger another reload, but
620
# this time we just raise an exception because we can't recover
621
for trans, name in vf._access._indices.itervalues():
623
self.assertRaises(errors.NoSuchFile, vf._get_record_map, keys)
624
self.assertEqual([2, 1, 1], reload_counter)
626
def test_get_record_stream_retries(self):
627
vf, reload_counter = self.make_vf_for_retrying()
628
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
629
record_stream = vf.get_record_stream(keys, 'topological', False)
630
record = record_stream.next()
631
self.assertEqual(('rev-1',), record.key)
632
self.assertEqual([0, 0, 0], reload_counter)
633
record = record_stream.next()
634
self.assertEqual(('rev-2',), record.key)
635
self.assertEqual([1, 1, 0], reload_counter)
636
record = record_stream.next()
637
self.assertEqual(('rev-3',), record.key)
638
self.assertEqual([1, 1, 0], reload_counter)
639
# Now delete all pack files, and see that we raise the right error
640
for trans, name in vf._access._indices.itervalues():
642
self.assertListRaises(errors.NoSuchFile,
643
vf.get_record_stream, keys, 'topological', False)
645
def test_iter_lines_added_or_present_in_keys_retries(self):
646
vf, reload_counter = self.make_vf_for_retrying()
647
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
648
# Unfortunately, iter_lines_added_or_present_in_keys iterates the
649
# result in random order (determined by the iteration order from a
650
# set()), so we don't have any solid way to trigger whether data is
651
# read before or after. However we tried to delete the middle node to
652
# exercise the code well.
653
# What we care about is that all lines are always yielded, but not
656
reload_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
657
self.assertEqual([1, 1, 0], reload_counter)
658
# Now do it again, to make sure the result is equivalent
659
plain_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
660
self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
661
self.assertEqual(plain_lines, reload_lines)
662
self.assertEqual(21, len(plain_lines))
663
# Now delete all pack files, and see that we raise the right error
664
for trans, name in vf._access._indices.itervalues():
666
self.assertListRaises(errors.NoSuchFile,
667
vf.iter_lines_added_or_present_in_keys, keys)
668
self.assertEqual([2, 1, 1], reload_counter)
670
def test_get_record_stream_yields_disk_sorted_order(self):
671
# if we get 'unordered' pick a semi-optimal order for reading. The
672
# order should be grouped by pack file, and then by position in file
673
repo = self.make_repository('test', format='pack-0.92')
675
self.addCleanup(repo.unlock)
676
repo.start_write_group()
678
vf.add_lines(('f-id', 'rev-5'), [('f-id', 'rev-4')], ['lines\n'])
679
vf.add_lines(('f-id', 'rev-1'), [], ['lines\n'])
680
vf.add_lines(('f-id', 'rev-2'), [('f-id', 'rev-1')], ['lines\n'])
681
repo.commit_write_group()
682
# We inserted them as rev-5, rev-1, rev-2, we should get them back in
684
stream = vf.get_record_stream([('f-id', 'rev-1'), ('f-id', 'rev-5'),
685
('f-id', 'rev-2')], 'unordered', False)
686
keys = [r.key for r in stream]
687
self.assertEqual([('f-id', 'rev-5'), ('f-id', 'rev-1'),
688
('f-id', 'rev-2')], keys)
689
repo.start_write_group()
690
vf.add_lines(('f-id', 'rev-4'), [('f-id', 'rev-3')], ['lines\n'])
691
vf.add_lines(('f-id', 'rev-3'), [('f-id', 'rev-2')], ['lines\n'])
692
vf.add_lines(('f-id', 'rev-6'), [('f-id', 'rev-5')], ['lines\n'])
693
repo.commit_write_group()
694
# Request in random order, to make sure the output order isn't based on
696
request_keys = set(('f-id', 'rev-%d' % i) for i in range(1, 7))
697
stream = vf.get_record_stream(request_keys, 'unordered', False)
698
keys = [r.key for r in stream]
699
# We want to get the keys back in disk order, but it doesn't matter
700
# which pack we read from first. So this can come back in 2 orders
701
alt1 = [('f-id', 'rev-%d' % i) for i in [4, 3, 6, 5, 1, 2]]
702
alt2 = [('f-id', 'rev-%d' % i) for i in [5, 1, 2, 4, 3, 6]]
703
if keys != alt1 and keys != alt2:
704
self.fail('Returned key order did not match either expected order.'
705
' expected %s or %s, not %s'
706
% (alt1, alt2, keys))
348
709
class LowLevelKnitDataTests(TestCase):
454
858
# Change 2 bytes in the middle to \xff
455
859
gz_txt = gz_txt[:10] + '\xff\xff' + gz_txt[12:]
456
860
transport = MockTransport([gz_txt])
457
access = _KnitAccess(transport, 'filename', None, None, False, False)
458
data = _KnitData(access=access)
459
records = [('rev-id-1', (None, 0, len(gz_txt)))]
461
self.assertRaises(errors.KnitCorrupt, data.read_records, records)
463
# read_records_iter_raw will notice if we request the wrong version.
464
self.assertRaises(errors.KnitCorrupt, list,
465
data.read_records_iter_raw(records))
861
access = _KnitKeyAccess(transport, ConstantMapper('filename'))
862
knit = KnitVersionedFiles(None, access)
863
records = [(('rev-id-1',), (('rev-id-1',), 0, len(gz_txt)))]
864
self.assertRaises(errors.KnitCorrupt, list,
865
knit._read_records_iter(records))
866
# read_records_iter_raw will barf on bad gz data
867
self.assertRaises(errors.KnitCorrupt, list,
868
knit._read_records_iter_raw(records))
468
871
class LowLevelKnitIndexTests(TestCase):
470
def get_knit_index(self, *args, **kwargs):
873
def get_knit_index(self, transport, name, mode):
874
mapper = ConstantMapper(name)
471
875
orig = knit._load_data
473
877
knit._load_data = orig
474
878
self.addCleanup(reset)
475
879
from bzrlib._knit_load_data_py import _load_data_py
476
880
knit._load_data = _load_data_py
477
return _KnitIndex(get_scope=lambda:None, *args, **kwargs)
479
def test_no_such_file(self):
480
transport = MockTransport()
482
self.assertRaises(NoSuchFile, self.get_knit_index,
483
transport, "filename", "r")
484
self.assertRaises(NoSuchFile, self.get_knit_index,
485
transport, "filename", "w", create=False)
881
allow_writes = lambda: 'w' in mode
882
return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
487
884
def test_create_file(self):
488
885
transport = MockTransport()
490
index = self.get_knit_index(transport, "filename", "w",
491
file_mode="wb", create=True)
493
("put_bytes_non_atomic",
494
("filename", index.HEADER), {"mode": "wb"}),
495
transport.calls.pop(0))
497
def test_delay_create_file(self):
498
transport = MockTransport()
500
index = self.get_knit_index(transport, "filename", "w",
501
create=True, file_mode="wb", create_parent_dir=True,
502
delay_create=True, dir_mode=0777)
503
self.assertEqual([], transport.calls)
505
index.add_versions([])
506
name, (filename, f), kwargs = transport.calls.pop(0)
507
self.assertEqual("put_file_non_atomic", name)
509
{"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
511
self.assertEqual("filename", filename)
512
self.assertEqual(index.HEADER, f.read())
514
index.add_versions([])
515
self.assertEqual(("append_bytes", ("filename", ""), {}),
516
transport.calls.pop(0))
886
index = self.get_knit_index(transport, "filename", "w")
888
call = transport.calls.pop(0)
889
# call[1][1] is a StringIO - we can't test it by simple equality.
890
self.assertEqual('put_file_non_atomic', call[0])
891
self.assertEqual('filename.kndx', call[1][0])
892
# With no history, _KndxIndex writes a new index:
893
self.assertEqual(_KndxIndex.HEADER,
894
call[1][1].getvalue())
895
self.assertEqual({'create_parent_dir': True}, call[2])
518
897
def test_read_utf8_version_id(self):
519
898
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
520
899
utf8_revision_id = unicode_revision_id.encode('utf-8')
521
900
transport = MockTransport([
523
902
'%s option 0 1 :' % (utf8_revision_id,)
525
904
index = self.get_knit_index(transport, "filename", "r")
526
# _KnitIndex is a private class, and deals in utf8 revision_ids, not
905
# _KndxIndex is a private class, and deals in utf8 revision_ids, not
527
906
# Unicode revision_ids.
528
self.assertTrue(index.has_version(utf8_revision_id))
529
self.assertFalse(index.has_version(unicode_revision_id))
907
self.assertEqual({(utf8_revision_id,):()},
908
index.get_parent_map(index.keys()))
909
self.assertFalse((unicode_revision_id,) in index.keys())
531
911
def test_read_utf8_parents(self):
532
912
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
533
913
utf8_revision_id = unicode_revision_id.encode('utf-8')
534
914
transport = MockTransport([
536
916
"version option 0 1 .%s :" % (utf8_revision_id,)
538
918
index = self.get_knit_index(transport, "filename", "r")
539
self.assertEqual((utf8_revision_id,),
540
index.get_parents_with_ghosts("version"))
919
self.assertEqual({("version",):((utf8_revision_id,),)},
920
index.get_parent_map(index.keys()))
542
922
def test_read_ignore_corrupted_lines(self):
543
923
transport = MockTransport([
546
926
"corrupted options 0 1 .b .c ",
547
927
"version options 0 1 :"
549
929
index = self.get_knit_index(transport, "filename", "r")
550
self.assertEqual(1, index.num_versions())
551
self.assertTrue(index.has_version("version"))
930
self.assertEqual(1, len(index.keys()))
931
self.assertEqual(set([("version",)]), index.keys())
553
933
def test_read_corrupted_header(self):
554
934
transport = MockTransport(['not a bzr knit index header\n'])
555
self.assertRaises(KnitHeaderError,
556
self.get_knit_index, transport, "filename", "r")
935
index = self.get_knit_index(transport, "filename", "r")
936
self.assertRaises(KnitHeaderError, index.keys)
558
938
def test_read_duplicate_entries(self):
559
939
transport = MockTransport([
561
941
"parent options 0 1 :",
562
942
"version options1 0 1 0 :",
563
943
"version options2 1 2 .other :",
564
944
"version options3 3 4 0 .other :"
566
946
index = self.get_knit_index(transport, "filename", "r")
567
self.assertEqual(2, index.num_versions())
947
self.assertEqual(2, len(index.keys()))
568
948
# check that the index used is the first one written. (Specific
569
949
# to KnitIndex style indices.
570
self.assertEqual("1", index._version_list_to_index(["version"]))
571
self.assertEqual((None, 3, 4), index.get_position("version"))
572
self.assertEqual(["options3"], index.get_options("version"))
573
self.assertEqual(("parent", "other"),
574
index.get_parents_with_ghosts("version"))
950
self.assertEqual("1", index._dictionary_compress([("version",)]))
951
self.assertEqual((("version",), 3, 4), index.get_position(("version",)))
952
self.assertEqual(["options3"], index.get_options(("version",)))
953
self.assertEqual({("version",):(("parent",), ("other",))},
954
index.get_parent_map([("version",)]))
576
956
def test_read_compressed_parents(self):
577
957
transport = MockTransport([
579
959
"a option 0 1 :",
580
960
"b option 0 1 0 :",
581
961
"c option 0 1 1 0 :",
583
963
index = self.get_knit_index(transport, "filename", "r")
584
self.assertEqual({"b":("a",), "c":("b", "a")},
585
index.get_parent_map(["b", "c"]))
964
self.assertEqual({("b",):(("a",),), ("c",):(("b",), ("a",))},
965
index.get_parent_map([("b",), ("c",)]))
587
967
def test_write_utf8_version_id(self):
588
968
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
589
969
utf8_revision_id = unicode_revision_id.encode('utf-8')
590
970
transport = MockTransport([
593
973
index = self.get_knit_index(transport, "filename", "r")
594
index.add_version(utf8_revision_id, ["option"], (None, 0, 1), [])
595
self.assertEqual(("append_bytes", ("filename",
596
"\n%s option 0 1 :" % (utf8_revision_id,)),
598
transport.calls.pop(0))
975
((utf8_revision_id,), ["option"], ((utf8_revision_id,), 0, 1), [])])
976
call = transport.calls.pop(0)
977
# call[1][1] is a StringIO - we can't test it by simple equality.
978
self.assertEqual('put_file_non_atomic', call[0])
979
self.assertEqual('filename.kndx', call[1][0])
980
# With no history, _KndxIndex writes a new index:
981
self.assertEqual(_KndxIndex.HEADER +
982
"\n%s option 0 1 :" % (utf8_revision_id,),
983
call[1][1].getvalue())
984
self.assertEqual({'create_parent_dir': True}, call[2])
600
986
def test_write_utf8_parents(self):
601
987
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
602
988
utf8_revision_id = unicode_revision_id.encode('utf-8')
603
989
transport = MockTransport([
606
index = self.get_knit_index(transport, "filename", "r")
607
index.add_version("version", ["option"], (None, 0, 1), [utf8_revision_id])
608
self.assertEqual(("append_bytes", ("filename",
609
"\nversion option 0 1 .%s :" % (utf8_revision_id,)),
611
transport.calls.pop(0))
613
def test_get_ancestry(self):
614
transport = MockTransport([
617
"b option 0 1 0 .e :",
618
"c option 0 1 1 0 :",
619
"d option 0 1 2 .f :"
621
index = self.get_knit_index(transport, "filename", "r")
623
self.assertEqual([], index.get_ancestry([]))
624
self.assertEqual(["a"], index.get_ancestry(["a"]))
625
self.assertEqual(["a", "b"], index.get_ancestry(["b"]))
626
self.assertEqual(["a", "b", "c"], index.get_ancestry(["c"]))
627
self.assertEqual(["a", "b", "c", "d"], index.get_ancestry(["d"]))
628
self.assertEqual(["a", "b"], index.get_ancestry(["a", "b"]))
629
self.assertEqual(["a", "b", "c"], index.get_ancestry(["a", "c"]))
631
self.assertRaises(RevisionNotPresent, index.get_ancestry, ["e"])
633
def test_get_ancestry_with_ghosts(self):
634
transport = MockTransport([
637
"b option 0 1 0 .e :",
638
"c option 0 1 0 .f .g :",
639
"d option 0 1 2 .h .j .k :"
641
index = self.get_knit_index(transport, "filename", "r")
643
self.assertEqual([], index.get_ancestry_with_ghosts([]))
644
self.assertEqual(["a"], index.get_ancestry_with_ghosts(["a"]))
645
self.assertEqual(["a", "e", "b"],
646
index.get_ancestry_with_ghosts(["b"]))
647
self.assertEqual(["a", "g", "f", "c"],
648
index.get_ancestry_with_ghosts(["c"]))
649
self.assertEqual(["a", "g", "f", "c", "k", "j", "h", "d"],
650
index.get_ancestry_with_ghosts(["d"]))
651
self.assertEqual(["a", "e", "b"],
652
index.get_ancestry_with_ghosts(["a", "b"]))
653
self.assertEqual(["a", "g", "f", "c"],
654
index.get_ancestry_with_ghosts(["a", "c"]))
992
index = self.get_knit_index(transport, "filename", "r")
994
(("version",), ["option"], (("version",), 0, 1), [(utf8_revision_id,)])])
995
call = transport.calls.pop(0)
996
# call[1][1] is a StringIO - we can't test it by simple equality.
997
self.assertEqual('put_file_non_atomic', call[0])
998
self.assertEqual('filename.kndx', call[1][0])
999
# With no history, _KndxIndex writes a new index:
1000
self.assertEqual(_KndxIndex.HEADER +
1001
"\nversion option 0 1 .%s :" % (utf8_revision_id,),
1002
call[1][1].getvalue())
1003
self.assertEqual({'create_parent_dir': True}, call[2])
1005
def test_keys(self):
1006
transport = MockTransport([
1009
index = self.get_knit_index(transport, "filename", "r")
1011
self.assertEqual(set(), index.keys())
1013
index.add_records([(("a",), ["option"], (("a",), 0, 1), [])])
1014
self.assertEqual(set([("a",)]), index.keys())
1016
index.add_records([(("a",), ["option"], (("a",), 0, 1), [])])
1017
self.assertEqual(set([("a",)]), index.keys())
1019
index.add_records([(("b",), ["option"], (("b",), 0, 1), [])])
1020
self.assertEqual(set([("a",), ("b",)]), index.keys())
1022
def add_a_b(self, index, random_id=None):
1024
if random_id is not None:
1025
kwargs["random_id"] = random_id
1027
(("a",), ["option"], (("a",), 0, 1), [("b",)]),
1028
(("a",), ["opt"], (("a",), 1, 2), [("c",)]),
1029
(("b",), ["option"], (("b",), 2, 3), [("a",)])
1032
def assertIndexIsAB(self, index):
1037
index.get_parent_map(index.keys()))
1038
self.assertEqual((("a",), 1, 2), index.get_position(("a",)))
1039
self.assertEqual((("b",), 2, 3), index.get_position(("b",)))
1040
self.assertEqual(["opt"], index.get_options(("a",)))
1042
def test_add_versions(self):
1043
transport = MockTransport([
1046
index = self.get_knit_index(transport, "filename", "r")
1049
call = transport.calls.pop(0)
1050
# call[1][1] is a StringIO - we can't test it by simple equality.
1051
self.assertEqual('put_file_non_atomic', call[0])
1052
self.assertEqual('filename.kndx', call[1][0])
1053
# With no history, _KndxIndex writes a new index:
655
1054
self.assertEqual(
656
["a", "g", "f", "c", "e", "b", "k", "j", "h", "d"],
657
index.get_ancestry_with_ghosts(["b", "d"]))
659
self.assertRaises(RevisionNotPresent,
660
index.get_ancestry_with_ghosts, ["e"])
662
def test_num_versions(self):
663
transport = MockTransport([
666
index = self.get_knit_index(transport, "filename", "r")
668
self.assertEqual(0, index.num_versions())
669
self.assertEqual(0, len(index))
671
index.add_version("a", ["option"], (None, 0, 1), [])
672
self.assertEqual(1, index.num_versions())
673
self.assertEqual(1, len(index))
675
index.add_version("a", ["option2"], (None, 1, 2), [])
676
self.assertEqual(1, index.num_versions())
677
self.assertEqual(1, len(index))
679
index.add_version("b", ["option"], (None, 0, 1), [])
680
self.assertEqual(2, index.num_versions())
681
self.assertEqual(2, len(index))
683
def test_get_versions(self):
684
transport = MockTransport([
687
index = self.get_knit_index(transport, "filename", "r")
689
self.assertEqual([], index.get_versions())
691
index.add_version("a", ["option"], (None, 0, 1), [])
692
self.assertEqual(["a"], index.get_versions())
694
index.add_version("a", ["option"], (None, 0, 1), [])
695
self.assertEqual(["a"], index.get_versions())
697
index.add_version("b", ["option"], (None, 0, 1), [])
698
self.assertEqual(["a", "b"], index.get_versions())
700
def test_add_version(self):
701
transport = MockTransport([
704
index = self.get_knit_index(transport, "filename", "r")
706
index.add_version("a", ["option"], (None, 0, 1), ["b"])
707
self.assertEqual(("append_bytes",
708
("filename", "\na option 0 1 .b :"),
709
{}), transport.calls.pop(0))
710
self.assertTrue(index.has_version("a"))
711
self.assertEqual(1, index.num_versions())
712
self.assertEqual((None, 0, 1), index.get_position("a"))
713
self.assertEqual(["option"], index.get_options("a"))
714
self.assertEqual(("b",), index.get_parents_with_ghosts("a"))
716
index.add_version("a", ["opt"], (None, 1, 2), ["c"])
717
self.assertEqual(("append_bytes",
718
("filename", "\na opt 1 2 .c :"),
719
{}), transport.calls.pop(0))
720
self.assertTrue(index.has_version("a"))
721
self.assertEqual(1, index.num_versions())
722
self.assertEqual((None, 1, 2), index.get_position("a"))
723
self.assertEqual(["opt"], index.get_options("a"))
724
self.assertEqual(("c",), index.get_parents_with_ghosts("a"))
726
index.add_version("b", ["option"], (None, 2, 3), ["a"])
727
self.assertEqual(("append_bytes",
728
("filename", "\nb option 2 3 0 :"),
729
{}), transport.calls.pop(0))
730
self.assertTrue(index.has_version("b"))
731
self.assertEqual(2, index.num_versions())
732
self.assertEqual((None, 2, 3), index.get_position("b"))
733
self.assertEqual(["option"], index.get_options("b"))
734
self.assertEqual(("a",), index.get_parents_with_ghosts("b"))
736
def test_add_versions(self):
737
transport = MockTransport([
740
index = self.get_knit_index(transport, "filename", "r")
743
("a", ["option"], (None, 0, 1), ["b"]),
744
("a", ["opt"], (None, 1, 2), ["c"]),
745
("b", ["option"], (None, 2, 3), ["a"])
747
self.assertEqual(("append_bytes", ("filename",
748
1056
"\na option 0 1 .b :"
749
1057
"\na opt 1 2 .c :"
751
), {}), transport.calls.pop(0))
752
self.assertTrue(index.has_version("a"))
753
self.assertTrue(index.has_version("b"))
754
self.assertEqual(2, index.num_versions())
755
self.assertEqual((None, 1, 2), index.get_position("a"))
756
self.assertEqual((None, 2, 3), index.get_position("b"))
757
self.assertEqual(["opt"], index.get_options("a"))
758
self.assertEqual(["option"], index.get_options("b"))
759
self.assertEqual(("c",), index.get_parents_with_ghosts("a"))
760
self.assertEqual(("a",), index.get_parents_with_ghosts("b"))
1058
"\nb option 2 3 0 :",
1059
call[1][1].getvalue())
1060
self.assertEqual({'create_parent_dir': True}, call[2])
1061
self.assertIndexIsAB(index)
762
1063
def test_add_versions_random_id_is_accepted(self):
763
1064
transport = MockTransport([
766
1067
index = self.get_knit_index(transport, "filename", "r")
769
("a", ["option"], (None, 0, 1), ["b"]),
770
("a", ["opt"], (None, 1, 2), ["c"]),
771
("b", ["option"], (None, 2, 3), ["a"])
1068
self.add_a_b(index, random_id=True)
774
1070
def test_delay_create_and_add_versions(self):
775
1071
transport = MockTransport()
777
index = self.get_knit_index(transport, "filename", "w",
778
create=True, file_mode="wb", create_parent_dir=True,
779
delay_create=True, dir_mode=0777)
1073
index = self.get_knit_index(transport, "filename", "w")
780
1075
self.assertEqual([], transport.calls)
783
("a", ["option"], (None, 0, 1), ["b"]),
784
("a", ["opt"], (None, 1, 2), ["c"]),
785
("b", ["option"], (None, 2, 3), ["a"])
787
name, (filename, f), kwargs = transport.calls.pop(0)
788
self.assertEqual("put_file_non_atomic", name)
790
{"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
792
self.assertEqual("filename", filename)
1078
#[ {"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
1080
# Two calls: one during which we load the existing index (and when its
1081
# missing create it), then a second where we write the contents out.
1082
self.assertEqual(2, len(transport.calls))
1083
call = transport.calls.pop(0)
1084
self.assertEqual('put_file_non_atomic', call[0])
1085
self.assertEqual('filename.kndx', call[1][0])
1086
# With no history, _KndxIndex writes a new index:
1087
self.assertEqual(_KndxIndex.HEADER, call[1][1].getvalue())
1088
self.assertEqual({'create_parent_dir': True}, call[2])
1089
call = transport.calls.pop(0)
1090
# call[1][1] is a StringIO - we can't test it by simple equality.
1091
self.assertEqual('put_file_non_atomic', call[0])
1092
self.assertEqual('filename.kndx', call[1][0])
1093
# With no history, _KndxIndex writes a new index:
795
1096
"\na option 0 1 .b :"
796
1097
"\na opt 1 2 .c :"
797
1098
"\nb option 2 3 0 :",
800
def test_has_version(self):
801
transport = MockTransport([
805
index = self.get_knit_index(transport, "filename", "r")
807
self.assertTrue(index.has_version("a"))
808
self.assertFalse(index.has_version("b"))
1099
call[1][1].getvalue())
1100
self.assertEqual({'create_parent_dir': True}, call[2])
1102
def assertTotalBuildSize(self, size, keys, positions):
1103
self.assertEqual(size,
1104
knit._get_total_build_size(None, keys, positions))
1106
def test__get_total_build_size(self):
1108
('a',): (('fulltext', False), (('a',), 0, 100), None),
1109
('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
1110
('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
1111
('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
1113
self.assertTotalBuildSize(100, [('a',)], positions)
1114
self.assertTotalBuildSize(121, [('b',)], positions)
1115
# c needs both a & b
1116
self.assertTotalBuildSize(156, [('c',)], positions)
1117
# we shouldn't count 'b' twice
1118
self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
1119
self.assertTotalBuildSize(133, [('d',)], positions)
1120
self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
810
1122
def test_get_position(self):
811
1123
transport = MockTransport([
813
1125
"a option 0 1 :",
814
1126
"b option 1 2 :"
816
1128
index = self.get_knit_index(transport, "filename", "r")
818
self.assertEqual((None, 0, 1), index.get_position("a"))
819
self.assertEqual((None, 1, 2), index.get_position("b"))
1130
self.assertEqual((("a",), 0, 1), index.get_position(("a",)))
1131
self.assertEqual((("b",), 1, 2), index.get_position(("b",)))
821
1133
def test_get_method(self):
822
1134
transport = MockTransport([
824
1136
"a fulltext,unknown 0 1 :",
825
1137
"b unknown,line-delta 1 2 :",
1268
def test_scan_unvalidated_index_not_implemented(self):
1269
transport = MockTransport()
1270
index = self.get_knit_index(transport, 'filename', 'r')
1272
NotImplementedError, index.scan_unvalidated_index,
1273
'dummy graph_index')
1275
NotImplementedError, index.get_missing_compression_parents)
987
1277
def test_short_line(self):
988
1278
transport = MockTransport([
990
1280
"a option 0 10 :",
991
1281
"b option 10 10 0", # This line isn't terminated, ignored
993
1283
index = self.get_knit_index(transport, "filename", "r")
994
self.assertEqual(['a'], index.get_versions())
1284
self.assertEqual(set([('a',)]), index.keys())
996
1286
def test_skip_incomplete_record(self):
997
1287
# A line with bogus data should just be skipped
998
1288
transport = MockTransport([
1000
1290
"a option 0 10 :",
1001
1291
"b option 10 10 0", # This line isn't terminated, ignored
1002
1292
"c option 20 10 0 :", # Properly terminated, and starts with '\n'
1004
1294
index = self.get_knit_index(transport, "filename", "r")
1005
self.assertEqual(['a', 'c'], index.get_versions())
1295
self.assertEqual(set([('a',), ('c',)]), index.keys())
1007
1297
def test_trailing_characters(self):
1008
1298
# A line with bogus data should just be skipped
1009
1299
transport = MockTransport([
1011
1301
"a option 0 10 :",
1012
1302
"b option 10 10 0 :a", # This line has extra trailing characters
1013
1303
"c option 20 10 0 :", # Properly terminated, and starts with '\n'
1015
1305
index = self.get_knit_index(transport, "filename", "r")
1016
self.assertEqual(['a', 'c'], index.get_versions())
1306
self.assertEqual(set([('a',), ('c',)]), index.keys())
1019
1309
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1021
1311
_test_needs_features = [CompiledKnitFeature]
1023
def get_knit_index(self, *args, **kwargs):
1313
def get_knit_index(self, transport, name, mode):
1314
mapper = ConstantMapper(name)
1024
1315
orig = knit._load_data
1026
1317
knit._load_data = orig
1027
1318
self.addCleanup(reset)
1028
from bzrlib._knit_load_data_c import _load_data_c
1319
from bzrlib._knit_load_data_pyx import _load_data_c
1029
1320
knit._load_data = _load_data_c
1030
return _KnitIndex(get_scope=lambda:None, *args, **kwargs)
1321
allow_writes = lambda: mode == 'w'
1322
return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
1325
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1327
def make_annotator(self):
1328
factory = knit.make_pack_factory(True, True, 1)
1329
vf = factory(self.get_transport())
1330
return knit._KnitAnnotator(vf)
1332
def test__expand_fulltext(self):
1333
ann = self.make_annotator()
1334
rev_key = ('rev-id',)
1335
ann._num_compression_children[rev_key] = 1
1336
res = ann._expand_record(rev_key, (('parent-id',),), None,
1337
['line1\n', 'line2\n'], ('fulltext', True))
1338
# The content object and text lines should be cached appropriately
1339
self.assertEqual(['line1\n', 'line2'], res)
1340
content_obj = ann._content_objects[rev_key]
1341
self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
1342
self.assertEqual(res, content_obj.text())
1343
self.assertEqual(res, ann._text_cache[rev_key])
1345
def test__expand_delta_comp_parent_not_available(self):
1346
# Parent isn't available yet, so we return nothing, but queue up this
1347
# node for later processing
1348
ann = self.make_annotator()
1349
rev_key = ('rev-id',)
1350
parent_key = ('parent-id',)
1351
record = ['0,1,1\n', 'new-line\n']
1352
details = ('line-delta', False)
1353
res = ann._expand_record(rev_key, (parent_key,), parent_key,
1355
self.assertEqual(None, res)
1356
self.assertTrue(parent_key in ann._pending_deltas)
1357
pending = ann._pending_deltas[parent_key]
1358
self.assertEqual(1, len(pending))
1359
self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
1361
def test__expand_record_tracks_num_children(self):
1362
ann = self.make_annotator()
1363
rev_key = ('rev-id',)
1364
rev2_key = ('rev2-id',)
1365
parent_key = ('parent-id',)
1366
record = ['0,1,1\n', 'new-line\n']
1367
details = ('line-delta', False)
1368
ann._num_compression_children[parent_key] = 2
1369
ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
1370
('fulltext', False))
1371
res = ann._expand_record(rev_key, (parent_key,), parent_key,
1373
self.assertEqual({parent_key: 1}, ann._num_compression_children)
1374
# Expanding the second child should remove the content object, and the
1375
# num_compression_children entry
1376
res = ann._expand_record(rev2_key, (parent_key,), parent_key,
1378
self.assertFalse(parent_key in ann._content_objects)
1379
self.assertEqual({}, ann._num_compression_children)
1380
# We should not cache the content_objects for rev2 and rev, because
1381
# they do not have compression children of their own.
1382
self.assertEqual({}, ann._content_objects)
1384
def test__expand_delta_records_blocks(self):
1385
ann = self.make_annotator()
1386
rev_key = ('rev-id',)
1387
parent_key = ('parent-id',)
1388
record = ['0,1,1\n', 'new-line\n']
1389
details = ('line-delta', True)
1390
ann._num_compression_children[parent_key] = 2
1391
ann._expand_record(parent_key, (), None,
1392
['line1\n', 'line2\n', 'line3\n'],
1393
('fulltext', False))
1394
ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
1395
self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
1396
ann._matching_blocks)
1397
rev2_key = ('rev2-id',)
1398
record = ['0,1,1\n', 'new-line\n']
1399
details = ('line-delta', False)
1400
ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
1401
self.assertEqual([(1, 1, 2), (3, 3, 0)],
1402
ann._matching_blocks[(rev2_key, parent_key)])
1404
def test__get_parent_ann_uses_matching_blocks(self):
1405
ann = self.make_annotator()
1406
rev_key = ('rev-id',)
1407
parent_key = ('parent-id',)
1408
parent_ann = [(parent_key,)]*3
1409
block_key = (rev_key, parent_key)
1410
ann._annotations_cache[parent_key] = parent_ann
1411
ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
1412
# We should not try to access any parent_lines content, because we know
1413
# we already have the matching blocks
1414
par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
1415
['1\n', '2\n', '3\n'], parent_key)
1416
self.assertEqual(parent_ann, par_ann)
1417
self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
1418
self.assertEqual({}, ann._matching_blocks)
1420
def test__process_pending(self):
1421
ann = self.make_annotator()
1422
rev_key = ('rev-id',)
1425
record = ['0,1,1\n', 'new-line\n']
1426
details = ('line-delta', False)
1427
p1_record = ['line1\n', 'line2\n']
1428
ann._num_compression_children[p1_key] = 1
1429
res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
1431
self.assertEqual(None, res)
1432
# self.assertTrue(p1_key in ann._pending_deltas)
1433
self.assertEqual({}, ann._pending_annotation)
1434
# Now insert p1, and we should be able to expand the delta
1435
res = ann._expand_record(p1_key, (), None, p1_record,
1436
('fulltext', False))
1437
self.assertEqual(p1_record, res)
1438
ann._annotations_cache[p1_key] = [(p1_key,)]*2
1439
res = ann._process_pending(p1_key)
1440
self.assertEqual([], res)
1441
self.assertFalse(p1_key in ann._pending_deltas)
1442
self.assertTrue(p2_key in ann._pending_annotation)
1443
self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
1444
ann._pending_annotation)
1445
# Now fill in parent 2, and pending annotation should be satisfied
1446
res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
1447
ann._annotations_cache[p2_key] = []
1448
res = ann._process_pending(p2_key)
1449
self.assertEqual([rev_key], res)
1450
self.assertEqual({}, ann._pending_annotation)
1451
self.assertEqual({}, ann._pending_deltas)
1453
def test_record_delta_removes_basis(self):
1454
ann = self.make_annotator()
1455
ann._expand_record(('parent-id',), (), None,
1456
['line1\n', 'line2\n'], ('fulltext', False))
1457
ann._num_compression_children['parent-id'] = 2
1459
def test_annotate_special_text(self):
1460
ann = self.make_annotator()
1462
rev1_key = ('rev-1',)
1463
rev2_key = ('rev-2',)
1464
rev3_key = ('rev-3',)
1465
spec_key = ('special:',)
1466
vf.add_lines(rev1_key, [], ['initial content\n'])
1467
vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
1470
vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
1473
spec_text = ('initial content\n'
1477
ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
1478
anns, lines = ann.annotate(spec_key)
1479
self.assertEqual([(rev1_key,),
1480
(rev2_key, rev3_key),
1484
self.assertEqualDiff(spec_text, ''.join(lines))
1033
1487
class KnitTests(TestCaseWithTransport):
1034
1488
"""Class containing knit test helper routines."""
1036
def make_test_knit(self, annotate=False, delay_create=False, index=None,
1037
name='test', delta=True, access_mode='w'):
1039
factory = KnitPlainFactory()
1043
index = _KnitIndex(get_transport('.'), name + INDEX_SUFFIX,
1044
access_mode, create=True, file_mode=None,
1045
create_parent_dir=False, delay_create=delay_create,
1046
dir_mode=None, get_scope=lambda:None)
1047
access = _KnitAccess(get_transport('.'), name + DATA_SUFFIX, None,
1048
None, delay_create, False)
1049
return KnitVersionedFile(name, get_transport('.'), factory=factory,
1050
create=True, delay_create=delay_create, index=index,
1051
access_method=access, delta=delta)
1053
def assertRecordContentEqual(self, knit, version_id, candidate_content):
1054
"""Assert that some raw record content matches the raw record content
1055
for a particular version_id in the given knit.
1057
index_memo = knit._index.get_position(version_id)
1058
record = (version_id, index_memo)
1059
[(_, expected_content, _)] = list(knit._data.read_records_iter_raw([record]))
1060
self.assertEqual(expected_content, candidate_content)
1063
class BasicKnitTests(KnitTests):
1065
def add_stock_one_and_one_a(self, k):
1066
k.add_lines('text-1', [], split_lines(TEXT_1))
1067
k.add_lines('text-1a', ['text-1'], split_lines(TEXT_1A))
1069
def test_knit_constructor(self):
1070
"""Construct empty k"""
1071
self.make_test_knit()
1073
def test_make_explicit_index(self):
1074
"""We can supply an index to use."""
1075
knit = KnitVersionedFile('test', get_transport('.'),
1076
index='strangelove', access_method="a")
1077
self.assertEqual(knit._index, 'strangelove')
1079
def test_knit_add(self):
1080
"""Store one text in knit and retrieve"""
1081
k = self.make_test_knit()
1082
k.add_lines('text-1', [], split_lines(TEXT_1))
1083
self.assertTrue(k.has_version('text-1'))
1084
self.assertEqualDiff(''.join(k.get_lines('text-1')), TEXT_1)
1086
def test_newline_empty_lines(self):
1087
# ensure that ["\n"] round trips ok.
1088
knit = self.make_test_knit()
1089
knit.add_lines('a', [], ["\n"])
1090
knit.add_lines_with_ghosts('b', [], ["\n"])
1091
self.assertEqual(["\n"], knit.get_lines('a'))
1092
self.assertEqual(["\n"], knit.get_lines('b'))
1093
self.assertEqual(['fulltext'], knit._index.get_options('a'))
1094
self.assertEqual(['fulltext'], knit._index.get_options('b'))
1095
knit.add_lines('c', ['a'], ["\n"])
1096
knit.add_lines_with_ghosts('d', ['b'], ["\n"])
1097
self.assertEqual(["\n"], knit.get_lines('c'))
1098
self.assertEqual(["\n"], knit.get_lines('d'))
1099
self.assertEqual(['line-delta'], knit._index.get_options('c'))
1100
self.assertEqual(['line-delta'], knit._index.get_options('d'))
1102
def test_empty_lines(self):
1103
# bizarrely, [] is not listed as having no-eol.
1104
knit = self.make_test_knit()
1105
knit.add_lines('a', [], [])
1106
knit.add_lines_with_ghosts('b', [], [])
1107
self.assertEqual([], knit.get_lines('a'))
1108
self.assertEqual([], knit.get_lines('b'))
1109
self.assertEqual(['fulltext'], knit._index.get_options('a'))
1110
self.assertEqual(['fulltext'], knit._index.get_options('b'))
1111
knit.add_lines('c', ['a'], [])
1112
knit.add_lines_with_ghosts('d', ['b'], [])
1113
self.assertEqual([], knit.get_lines('c'))
1114
self.assertEqual([], knit.get_lines('d'))
1115
self.assertEqual(['line-delta'], knit._index.get_options('c'))
1116
self.assertEqual(['line-delta'], knit._index.get_options('d'))
1118
def test_knit_reload(self):
1119
# test that the content in a reloaded knit is correct
1120
k = self.make_test_knit()
1121
k.add_lines('text-1', [], split_lines(TEXT_1))
1123
k2 = make_file_knit('test', get_transport('.'), access_mode='r',
1124
factory=KnitPlainFactory(), create=True)
1125
self.assertTrue(k2.has_version('text-1'))
1126
self.assertEqualDiff(''.join(k2.get_lines('text-1')), TEXT_1)
1128
def test_knit_several(self):
1129
"""Store several texts in a knit"""
1130
k = self.make_test_knit()
1131
k.add_lines('text-1', [], split_lines(TEXT_1))
1132
k.add_lines('text-2', [], split_lines(TEXT_2))
1133
self.assertEqualDiff(''.join(k.get_lines('text-1')), TEXT_1)
1134
self.assertEqualDiff(''.join(k.get_lines('text-2')), TEXT_2)
1136
def test_repeated_add(self):
1137
"""Knit traps attempt to replace existing version"""
1138
k = self.make_test_knit()
1139
k.add_lines('text-1', [], split_lines(TEXT_1))
1140
self.assertRaises(RevisionAlreadyPresent,
1142
'text-1', [], split_lines(TEXT_1))
1144
def test_empty(self):
1145
k = self.make_test_knit(True)
1146
k.add_lines('text-1', [], [])
1147
self.assertEquals(k.get_lines('text-1'), [])
1149
def test_incomplete(self):
1150
"""Test if texts without a ending line-end can be inserted and
1152
k = make_file_knit('test', get_transport('.'), delta=False, create=True)
1153
k.add_lines('text-1', [], ['a\n', 'b' ])
1154
k.add_lines('text-2', ['text-1'], ['a\rb\n', 'b\n'])
1155
# reopening ensures maximum room for confusion
1156
k = make_file_knit('test', get_transport('.'), delta=False, create=True)
1157
self.assertEquals(k.get_lines('text-1'), ['a\n', 'b' ])
1158
self.assertEquals(k.get_lines('text-2'), ['a\rb\n', 'b\n'])
1160
def test_delta(self):
1161
"""Expression of knit delta as lines"""
1162
k = self.make_test_knit()
1163
td = list(line_delta(TEXT_1.splitlines(True),
1164
TEXT_1A.splitlines(True)))
1165
self.assertEqualDiff(''.join(td), delta_1_1a)
1166
out = apply_line_delta(TEXT_1.splitlines(True), td)
1167
self.assertEqualDiff(''.join(out), TEXT_1A)
1169
def test_add_with_parents(self):
1170
"""Store in knit with parents"""
1171
k = self.make_test_knit()
1172
self.add_stock_one_and_one_a(k)
1173
self.assertEqual({'text-1':(), 'text-1a':('text-1',)},
1174
k.get_parent_map(['text-1', 'text-1a']))
1176
def test_ancestry(self):
1177
"""Store in knit with parents"""
1178
k = self.make_test_knit()
1179
self.add_stock_one_and_one_a(k)
1180
self.assertEquals(set(k.get_ancestry(['text-1a'])), set(['text-1a', 'text-1']))
1182
def test_add_delta(self):
1183
"""Store in knit with parents"""
1184
k = self.make_test_knit(annotate=False)
1185
self.add_stock_one_and_one_a(k)
1186
self.assertEqualDiff(''.join(k.get_lines('text-1a')), TEXT_1A)
1188
def test_add_delta_knit_graph_index(self):
1189
"""Does adding work with a KnitGraphIndex."""
1190
index = InMemoryGraphIndex(2)
1191
knit_index = KnitGraphIndex(index, add_callback=index.add_nodes,
1193
k = self.make_test_knit(annotate=True, index=knit_index)
1194
self.add_stock_one_and_one_a(k)
1195
self.assertEqualDiff(''.join(k.get_lines('text-1a')), TEXT_1A)
1196
# check the index had the right data added.
1197
self.assertEqual(set([
1198
(index, ('text-1', ), ' 0 127', ((), ())),
1199
(index, ('text-1a', ), ' 127 140', ((('text-1', ),), (('text-1', ),))),
1200
]), set(index.iter_all_entries()))
1201
# we should not have a .kndx file
1202
self.assertFalse(get_transport('.').has('test.kndx'))
1204
def test_annotate(self):
1206
k = self.make_test_knit(annotate=True, name='knit')
1207
self.insert_and_test_small_annotate(k)
1209
def insert_and_test_small_annotate(self, k):
1210
"""test annotation with k works correctly."""
1211
k.add_lines('text-1', [], ['a\n', 'b\n'])
1212
k.add_lines('text-2', ['text-1'], ['a\n', 'c\n'])
1214
origins = k.annotate('text-2')
1215
self.assertEquals(origins[0], ('text-1', 'a\n'))
1216
self.assertEquals(origins[1], ('text-2', 'c\n'))
1218
def test_annotate_fulltext(self):
1220
k = self.make_test_knit(annotate=True, name='knit', delta=False)
1221
self.insert_and_test_small_annotate(k)
1223
def test_annotate_merge_1(self):
1224
k = self.make_test_knit(True)
1225
k.add_lines('text-a1', [], ['a\n', 'b\n'])
1226
k.add_lines('text-a2', [], ['d\n', 'c\n'])
1227
k.add_lines('text-am', ['text-a1', 'text-a2'], ['d\n', 'b\n'])
1228
origins = k.annotate('text-am')
1229
self.assertEquals(origins[0], ('text-a2', 'd\n'))
1230
self.assertEquals(origins[1], ('text-a1', 'b\n'))
1232
def test_annotate_merge_2(self):
1233
k = self.make_test_knit(True)
1234
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1235
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1236
k.add_lines('text-am', ['text-a1', 'text-a2'], ['a\n', 'y\n', 'c\n'])
1237
origins = k.annotate('text-am')
1238
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1239
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1240
self.assertEquals(origins[2], ('text-a1', 'c\n'))
1242
def test_annotate_merge_9(self):
1243
k = self.make_test_knit(True)
1244
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1245
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1246
k.add_lines('text-am', ['text-a1', 'text-a2'], ['k\n', 'y\n', 'c\n'])
1247
origins = k.annotate('text-am')
1248
self.assertEquals(origins[0], ('text-am', 'k\n'))
1249
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1250
self.assertEquals(origins[2], ('text-a1', 'c\n'))
1252
def test_annotate_merge_3(self):
1253
k = self.make_test_knit(True)
1254
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1255
k.add_lines('text-a2', [] ,['x\n', 'y\n', 'z\n'])
1256
k.add_lines('text-am', ['text-a1', 'text-a2'], ['k\n', 'y\n', 'z\n'])
1257
origins = k.annotate('text-am')
1258
self.assertEquals(origins[0], ('text-am', 'k\n'))
1259
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1260
self.assertEquals(origins[2], ('text-a2', 'z\n'))
1262
def test_annotate_merge_4(self):
1263
k = self.make_test_knit(True)
1264
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1265
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1266
k.add_lines('text-a3', ['text-a1'], ['a\n', 'b\n', 'p\n'])
1267
k.add_lines('text-am', ['text-a2', 'text-a3'], ['a\n', 'b\n', 'z\n'])
1268
origins = k.annotate('text-am')
1269
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1270
self.assertEquals(origins[1], ('text-a1', 'b\n'))
1271
self.assertEquals(origins[2], ('text-a2', 'z\n'))
1273
def test_annotate_merge_5(self):
1274
k = self.make_test_knit(True)
1275
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1276
k.add_lines('text-a2', [], ['d\n', 'e\n', 'f\n'])
1277
k.add_lines('text-a3', [], ['x\n', 'y\n', 'z\n'])
1278
k.add_lines('text-am',
1279
['text-a1', 'text-a2', 'text-a3'],
1280
['a\n', 'e\n', 'z\n'])
1281
origins = k.annotate('text-am')
1282
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1283
self.assertEquals(origins[1], ('text-a2', 'e\n'))
1284
self.assertEquals(origins[2], ('text-a3', 'z\n'))
1286
def test_annotate_file_cherry_pick(self):
1287
k = self.make_test_knit(True)
1288
k.add_lines('text-1', [], ['a\n', 'b\n', 'c\n'])
1289
k.add_lines('text-2', ['text-1'], ['d\n', 'e\n', 'f\n'])
1290
k.add_lines('text-3', ['text-2', 'text-1'], ['a\n', 'b\n', 'c\n'])
1291
origins = k.annotate('text-3')
1292
self.assertEquals(origins[0], ('text-1', 'a\n'))
1293
self.assertEquals(origins[1], ('text-1', 'b\n'))
1294
self.assertEquals(origins[2], ('text-1', 'c\n'))
1296
def test_reannotate(self):
1297
k1 = make_file_knit('knit1', get_transport('.'),
1298
factory=KnitAnnotateFactory(), create=True)
1300
k1.add_lines('text-a', [], ['a\n', 'b\n'])
1302
k1.add_lines('text-b', ['text-a'], ['a\n', 'c\n'])
1304
k2 = make_file_knit('test2', get_transport('.'),
1305
factory=KnitAnnotateFactory(), create=True)
1306
k2.insert_record_stream(k1.get_record_stream(k1.versions(),
1307
'unordered', False))
1310
k1.add_lines('text-X', ['text-b'], ['a\n', 'b\n'])
1312
k2.add_lines('text-c', ['text-b'], ['z\n', 'c\n'])
1314
k2.add_lines('text-Y', ['text-b'], ['b\n', 'c\n'])
1316
# test-c will have index 3
1317
k1.insert_record_stream(k2.get_record_stream(['text-c'],
1318
'unordered', False))
1320
lines = k1.get_lines('text-c')
1321
self.assertEquals(lines, ['z\n', 'c\n'])
1323
origins = k1.annotate('text-c')
1324
self.assertEquals(origins[0], ('text-c', 'z\n'))
1325
self.assertEquals(origins[1], ('text-b', 'c\n'))
1327
def test_get_line_delta_texts(self):
1328
"""Make sure we can call get_texts on text with reused line deltas"""
1329
k1 = make_file_knit('test1', get_transport('.'),
1330
factory=KnitPlainFactory(), create=True)
1335
parents = ['%d' % (t-1)]
1336
k1.add_lines('%d' % t, parents, ['hello\n'] * t)
1337
k1.get_texts(('%d' % t) for t in range(3))
1339
def test_iter_lines_reads_in_order(self):
1340
instrumented_t = get_transport('trace+memory:///')
1341
k1 = make_file_knit('id', instrumented_t, create=True, delta=True)
1342
self.assertEqual([('get', 'id.kndx',)], instrumented_t._activity)
1343
# add texts with no required ordering
1344
k1.add_lines('base', [], ['text\n'])
1345
k1.add_lines('base2', [], ['text2\n'])
1346
# clear the logged activity, but preserve the list instance in case of
1347
# clones pointing at it.
1348
del instrumented_t._activity[:]
1349
# request a last-first iteration
1350
results = list(k1.iter_lines_added_or_present_in_versions(
1353
[('readv', 'id.knit', [(0, 87), (87, 89)], False, None)],
1354
instrumented_t._activity)
1355
self.assertEqual([('text\n', 'base'), ('text2\n', 'base2')], results)
1357
def test_knit_format(self):
1358
# this tests that a new knit index file has the expected content
1359
# and that is writes the data we expect as records are added.
1360
knit = self.make_test_knit(True)
1361
# Now knit files are not created until we first add data to them
1362
self.assertFileEqual("# bzr knit index 8\n", 'test.kndx')
1363
knit.add_lines_with_ghosts('revid', ['a_ghost'], ['a\n'])
1364
self.assertFileEqual(
1365
"# bzr knit index 8\n"
1367
"revid fulltext 0 84 .a_ghost :",
1369
knit.add_lines_with_ghosts('revid2', ['revid'], ['a\n'])
1370
self.assertFileEqual(
1371
"# bzr knit index 8\n"
1372
"\nrevid fulltext 0 84 .a_ghost :"
1373
"\nrevid2 line-delta 84 82 0 :",
1375
# we should be able to load this file again
1376
knit = make_file_knit('test', get_transport('.'), access_mode='r')
1377
self.assertEqual(['revid', 'revid2'], knit.versions())
1378
# write a short write to the file and ensure that its ignored
1379
indexfile = file('test.kndx', 'ab')
1380
indexfile.write('\nrevid3 line-delta 166 82 1 2 3 4 5 .phwoar:demo ')
1382
# we should be able to load this file again
1383
knit = make_file_knit('test', get_transport('.'), access_mode='w')
1384
self.assertEqual(['revid', 'revid2'], knit.versions())
1385
# and add a revision with the same id the failed write had
1386
knit.add_lines('revid3', ['revid2'], ['a\n'])
1387
# and when reading it revid3 should now appear.
1388
knit = make_file_knit('test', get_transport('.'), access_mode='r')
1389
self.assertEqual(['revid', 'revid2', 'revid3'], knit.versions())
1390
self.assertEqual({'revid3':('revid2',)}, knit.get_parent_map(['revid3']))
1392
def test_delay_create(self):
1393
"""Test that passing delay_create=True creates files late"""
1394
knit = self.make_test_knit(annotate=True, delay_create=True)
1395
self.failIfExists('test.knit')
1396
self.failIfExists('test.kndx')
1397
knit.add_lines_with_ghosts('revid', ['a_ghost'], ['a\n'])
1398
self.failUnlessExists('test.knit')
1399
self.assertFileEqual(
1400
"# bzr knit index 8\n"
1402
"revid fulltext 0 84 .a_ghost :",
1405
def test_create_parent_dir(self):
1406
"""create_parent_dir can create knits in nonexistant dirs"""
1407
# Has no effect if we don't set 'delay_create'
1408
trans = get_transport('.')
1409
self.assertRaises(NoSuchFile, make_file_knit, 'dir/test',
1410
trans, access_mode='w', factory=None,
1411
create=True, create_parent_dir=True)
1412
# Nothing should have changed yet
1413
knit = make_file_knit('dir/test', trans, access_mode='w',
1414
factory=None, create=True,
1415
create_parent_dir=True,
1417
self.failIfExists('dir/test.knit')
1418
self.failIfExists('dir/test.kndx')
1419
self.failIfExists('dir')
1420
knit.add_lines('revid', [], ['a\n'])
1421
self.failUnlessExists('dir')
1422
self.failUnlessExists('dir/test.knit')
1423
self.assertFileEqual(
1424
"# bzr knit index 8\n"
1426
"revid fulltext 0 84 :",
1429
def test_create_mode_700(self):
1430
trans = get_transport('.')
1431
if not trans._can_roundtrip_unix_modebits():
1432
# Can't roundtrip, so no need to run this test
1434
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1435
create=True, create_parent_dir=True, delay_create=True,
1436
file_mode=0600, dir_mode=0700)
1437
knit.add_lines('revid', [], ['a\n'])
1438
self.assertTransportMode(trans, 'dir', 0700)
1439
self.assertTransportMode(trans, 'dir/test.knit', 0600)
1440
self.assertTransportMode(trans, 'dir/test.kndx', 0600)
1442
def test_create_mode_770(self):
1443
trans = get_transport('.')
1444
if not trans._can_roundtrip_unix_modebits():
1445
# Can't roundtrip, so no need to run this test
1447
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1448
create=True, create_parent_dir=True, delay_create=True,
1449
file_mode=0660, dir_mode=0770)
1450
knit.add_lines('revid', [], ['a\n'])
1451
self.assertTransportMode(trans, 'dir', 0770)
1452
self.assertTransportMode(trans, 'dir/test.knit', 0660)
1453
self.assertTransportMode(trans, 'dir/test.kndx', 0660)
1455
def test_create_mode_777(self):
1456
trans = get_transport('.')
1457
if not trans._can_roundtrip_unix_modebits():
1458
# Can't roundtrip, so no need to run this test
1460
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1461
create=True, create_parent_dir=True, delay_create=True,
1462
file_mode=0666, dir_mode=0777)
1463
knit.add_lines('revid', [], ['a\n'])
1464
self.assertTransportMode(trans, 'dir', 0777)
1465
self.assertTransportMode(trans, 'dir/test.knit', 0666)
1466
self.assertTransportMode(trans, 'dir/test.kndx', 0666)
1468
def test_plan_merge(self):
1469
my_knit = self.make_test_knit(annotate=True)
1470
my_knit.add_lines('text1', [], split_lines(TEXT_1))
1471
my_knit.add_lines('text1a', ['text1'], split_lines(TEXT_1A))
1472
my_knit.add_lines('text1b', ['text1'], split_lines(TEXT_1B))
1473
plan = list(my_knit.plan_merge('text1a', 'text1b'))
1474
for plan_line, expected_line in zip(plan, AB_MERGE):
1475
self.assertEqual(plan_line, expected_line)
1478
class GetDataStreamTests(KnitTests):
1479
"""Tests for get_data_stream."""
1481
def test_get_stream_empty(self):
1482
"""Get a data stream for an empty knit file."""
1483
k1 = self.make_test_knit()
1484
format, data_list, reader_callable = k1.get_data_stream([])
1485
self.assertEqual('knit-plain', format)
1486
self.assertEqual([], data_list)
1487
content = reader_callable(None)
1488
self.assertEqual('', content)
1489
self.assertIsInstance(content, str)
1491
def test_get_stream_one_version(self):
1492
"""Get a data stream for a single record out of a knit containing just
1495
k1 = self.make_test_knit()
1497
('text-a', [], TEXT_1),
1499
expected_data_list = [
1500
# version, options, length, parents
1501
('text-a', ['fulltext'], 122, ()),
1503
for version_id, parents, lines in test_data:
1504
k1.add_lines(version_id, parents, split_lines(lines))
1506
format, data_list, reader_callable = k1.get_data_stream(['text-a'])
1507
self.assertEqual('knit-plain', format)
1508
self.assertEqual(expected_data_list, data_list)
1509
# There's only one record in the knit, so the content should be the
1510
# entire knit data file's contents.
1511
self.assertEqual(k1.transport.get_bytes(k1._data._access._filename),
1512
reader_callable(None))
1514
def test_get_stream_get_one_version_of_many(self):
1515
"""Get a data stream for just one version out of a knit containing many
1518
k1 = self.make_test_knit()
1519
# Insert the same data as test_knit_join, as they seem to cover a range
1520
# of cases (no parents, one parent, multiple parents).
1522
('text-a', [], TEXT_1),
1523
('text-b', ['text-a'], TEXT_1),
1524
('text-c', [], TEXT_1),
1525
('text-d', ['text-c'], TEXT_1),
1526
('text-m', ['text-b', 'text-d'], TEXT_1),
1528
expected_data_list = [
1529
# version, options, length, parents
1530
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1532
for version_id, parents, lines in test_data:
1533
k1.add_lines(version_id, parents, split_lines(lines))
1535
format, data_list, reader_callable = k1.get_data_stream(['text-m'])
1536
self.assertEqual('knit-plain', format)
1537
self.assertEqual(expected_data_list, data_list)
1538
self.assertRecordContentEqual(k1, 'text-m', reader_callable(None))
1540
def test_get_data_stream_unordered_index(self):
1541
"""Get a data stream when the knit index reports versions out of order.
1543
https://bugs.launchpad.net/bzr/+bug/164637
1545
k1 = self.make_test_knit()
1547
('text-a', [], TEXT_1),
1548
('text-b', ['text-a'], TEXT_1),
1549
('text-c', [], TEXT_1),
1550
('text-d', ['text-c'], TEXT_1),
1551
('text-m', ['text-b', 'text-d'], TEXT_1),
1553
for version_id, parents, lines in test_data:
1554
k1.add_lines(version_id, parents, split_lines(lines))
1555
# monkey-patch versions method to return out of order, as if coming
1556
# from multiple independently indexed packs
1557
original_versions = k1.versions
1558
k1.versions = lambda: reversed(original_versions())
1559
expected_data_list = [
1560
('text-a', ['fulltext'], 122, ()),
1561
('text-b', ['line-delta'], 84, ('text-a',))]
1562
# now check the fulltext is first and the delta second
1563
format, data_list, _ = k1.get_data_stream(['text-a', 'text-b'])
1564
self.assertEqual('knit-plain', format)
1565
self.assertEqual(expected_data_list, data_list)
1566
# and that's true if we ask for them in the opposite order too
1567
format, data_list, _ = k1.get_data_stream(['text-b', 'text-a'])
1568
self.assertEqual(expected_data_list, data_list)
1569
# also try requesting more versions
1570
format, data_list, _ = k1.get_data_stream([
1571
'text-m', 'text-b', 'text-a'])
1573
('text-a', ['fulltext'], 122, ()),
1574
('text-b', ['line-delta'], 84, ('text-a',)),
1575
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1578
def test_get_stream_ghost_parent(self):
1579
"""Get a data stream for a version with a ghost parent."""
1580
k1 = self.make_test_knit()
1582
k1.add_lines('text-a', [], split_lines(TEXT_1))
1583
k1.add_lines_with_ghosts('text-b', ['text-a', 'text-ghost'],
1584
split_lines(TEXT_1))
1586
expected_data_list = [
1587
# version, options, length, parents
1588
('text-b', ['line-delta'], 84, ('text-a', 'text-ghost')),
1591
format, data_list, reader_callable = k1.get_data_stream(['text-b'])
1592
self.assertEqual('knit-plain', format)
1593
self.assertEqual(expected_data_list, data_list)
1594
self.assertRecordContentEqual(k1, 'text-b', reader_callable(None))
1596
def test_get_stream_get_multiple_records(self):
1597
"""Get a stream for multiple records of a knit."""
1598
k1 = self.make_test_knit()
1599
# Insert the same data as test_knit_join, as they seem to cover a range
1600
# of cases (no parents, one parent, multiple parents).
1602
('text-a', [], TEXT_1),
1603
('text-b', ['text-a'], TEXT_1),
1604
('text-c', [], TEXT_1),
1605
('text-d', ['text-c'], TEXT_1),
1606
('text-m', ['text-b', 'text-d'], TEXT_1),
1608
for version_id, parents, lines in test_data:
1609
k1.add_lines(version_id, parents, split_lines(lines))
1611
# This test is actually a bit strict as the order in which they're
1612
# returned is not defined. This matches the current (deterministic)
1614
expected_data_list = [
1615
# version, options, length, parents
1616
('text-d', ['line-delta'], 84, ('text-c',)),
1617
('text-b', ['line-delta'], 84, ('text-a',)),
1619
# Note that even though we request the revision IDs in a particular
1620
# order, the data stream may return them in any order it likes. In this
1621
# case, they'll be in the order they were inserted into the knit.
1622
format, data_list, reader_callable = k1.get_data_stream(
1623
['text-d', 'text-b'])
1624
self.assertEqual('knit-plain', format)
1625
self.assertEqual(expected_data_list, data_list)
1626
# must match order they're returned
1627
self.assertRecordContentEqual(k1, 'text-d', reader_callable(84))
1628
self.assertRecordContentEqual(k1, 'text-b', reader_callable(84))
1629
self.assertEqual('', reader_callable(None),
1630
"There should be no more bytes left to read.")
1632
def test_get_stream_all(self):
1633
"""Get a data stream for all the records in a knit.
1635
This exercises fulltext records, line-delta records, records with
1636
various numbers of parents, and reading multiple records out of the
1637
callable. These cases ought to all be exercised individually by the
1638
other test_get_stream_* tests; this test is basically just paranoia.
1640
k1 = self.make_test_knit()
1641
# Insert the same data as test_knit_join, as they seem to cover a range
1642
# of cases (no parents, one parent, multiple parents).
1644
('text-a', [], TEXT_1),
1645
('text-b', ['text-a'], TEXT_1),
1646
('text-c', [], TEXT_1),
1647
('text-d', ['text-c'], TEXT_1),
1648
('text-m', ['text-b', 'text-d'], TEXT_1),
1650
for version_id, parents, lines in test_data:
1651
k1.add_lines(version_id, parents, split_lines(lines))
1653
# This test is actually a bit strict as the order in which they're
1654
# returned is not defined. This matches the current (deterministic)
1656
expected_data_list = [
1657
# version, options, length, parents
1658
('text-a', ['fulltext'], 122, ()),
1659
('text-b', ['line-delta'], 84, ('text-a',)),
1660
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1661
('text-c', ['fulltext'], 121, ()),
1662
('text-d', ['line-delta'], 84, ('text-c',)),
1664
format, data_list, reader_callable = k1.get_data_stream(
1665
['text-a', 'text-b', 'text-c', 'text-d', 'text-m'])
1666
self.assertEqual('knit-plain', format)
1667
self.assertEqual(expected_data_list, data_list)
1668
for version_id, options, length, parents in expected_data_list:
1669
bytes = reader_callable(length)
1670
self.assertRecordContentEqual(k1, version_id, bytes)
1673
class InsertDataStreamTests(KnitTests):
1674
"""Tests for insert_data_stream."""
1676
def assertKnitFilesEqual(self, knit1, knit2):
1677
"""Assert that the contents of the index and data files of two knits are
1681
knit1.transport.get_bytes(knit1._data._access._filename),
1682
knit2.transport.get_bytes(knit2._data._access._filename))
1684
knit1.transport.get_bytes(knit1._index._filename),
1685
knit2.transport.get_bytes(knit2._index._filename))
1687
def assertKnitValuesEqual(self, left, right):
1688
"""Assert that the texts, annotations and graph of left and right are
1691
self.assertEqual(set(left.versions()), set(right.versions()))
1692
for version in left.versions():
1693
self.assertEqual(left.get_parents_with_ghosts(version),
1694
right.get_parents_with_ghosts(version))
1695
self.assertEqual(left.get_lines(version),
1696
right.get_lines(version))
1697
self.assertEqual(left.annotate(version),
1698
right.annotate(version))
1700
def test_empty_stream(self):
1701
"""Inserting a data stream with no records should not put any data into
1704
k1 = self.make_test_knit()
1705
k1.insert_data_stream(
1706
(k1.get_format_signature(), [], lambda ignored: ''))
1707
self.assertEqual('', k1.transport.get_bytes(k1._data._access._filename),
1708
"The .knit should be completely empty.")
1709
self.assertEqual(k1._index.HEADER,
1710
k1.transport.get_bytes(k1._index._filename),
1711
"The .kndx should have nothing apart from the header.")
1713
def test_one_record(self):
1714
"""Inserting a data stream with one record from a knit with one record
1715
results in byte-identical files.
1717
source = self.make_test_knit(name='source')
1718
source.add_lines('text-a', [], split_lines(TEXT_1))
1719
data_stream = source.get_data_stream(['text-a'])
1720
target = self.make_test_knit(name='target')
1721
target.insert_data_stream(data_stream)
1722
self.assertKnitFilesEqual(source, target)
1724
def test_annotated_stream_into_unannotated_knit(self):
1725
"""Inserting an annotated datastream to an unannotated knit works."""
1726
# case one - full texts.
1727
source = self.make_test_knit(name='source', annotate=True)
1728
target = self.make_test_knit(name='target', annotate=False)
1729
source.add_lines('text-a', [], split_lines(TEXT_1))
1730
target.insert_data_stream(source.get_data_stream(['text-a']))
1731
self.assertKnitValuesEqual(source, target)
1732
# case two - deltas.
1733
source.add_lines('text-b', ['text-a'], split_lines(TEXT_2))
1734
target.insert_data_stream(source.get_data_stream(['text-b']))
1735
self.assertKnitValuesEqual(source, target)
1737
def test_unannotated_stream_into_annotated_knit(self):
1738
"""Inserting an unannotated datastream to an annotated knit works."""
1739
# case one - full texts.
1740
source = self.make_test_knit(name='source', annotate=False)
1741
target = self.make_test_knit(name='target', annotate=True)
1742
source.add_lines('text-a', [], split_lines(TEXT_1))
1743
target.insert_data_stream(source.get_data_stream(['text-a']))
1744
self.assertKnitValuesEqual(source, target)
1745
# case two - deltas.
1746
source.add_lines('text-b', ['text-a'], split_lines(TEXT_2))
1747
target.insert_data_stream(source.get_data_stream(['text-b']))
1748
self.assertKnitValuesEqual(source, target)
1750
def test_records_already_present(self):
1751
"""Insert a data stream where some records are alreday present in the
1752
target, and some not. Only the new records are inserted.
1754
source = self.make_test_knit(name='source')
1755
target = self.make_test_knit(name='target')
1756
# Insert 'text-a' into both source and target
1757
source.add_lines('text-a', [], split_lines(TEXT_1))
1758
target.insert_data_stream(source.get_data_stream(['text-a']))
1759
# Insert 'text-b' into just the source.
1760
source.add_lines('text-b', ['text-a'], split_lines(TEXT_1))
1761
# Get a data stream of both text-a and text-b, and insert it.
1762
data_stream = source.get_data_stream(['text-a', 'text-b'])
1763
target.insert_data_stream(data_stream)
1764
# The source and target will now be identical. This means the text-a
1765
# record was not added a second time.
1766
self.assertKnitFilesEqual(source, target)
1768
def test_multiple_records(self):
1769
"""Inserting a data stream of all records from a knit with multiple
1770
records results in byte-identical files.
1772
source = self.make_test_knit(name='source')
1773
source.add_lines('text-a', [], split_lines(TEXT_1))
1774
source.add_lines('text-b', ['text-a'], split_lines(TEXT_1))
1775
source.add_lines('text-c', [], split_lines(TEXT_1))
1776
data_stream = source.get_data_stream(['text-a', 'text-b', 'text-c'])
1778
target = self.make_test_knit(name='target')
1779
target.insert_data_stream(data_stream)
1781
self.assertKnitFilesEqual(source, target)
1783
def test_ghost_parent(self):
1784
"""Insert a data stream with a record that has a ghost parent."""
1785
# Make a knit with a record, text-a, that has a ghost parent.
1786
source = self.make_test_knit(name='source')
1787
source.add_lines_with_ghosts('text-a', ['text-ghost'],
1788
split_lines(TEXT_1))
1789
data_stream = source.get_data_stream(['text-a'])
1791
target = self.make_test_knit(name='target')
1792
target.insert_data_stream(data_stream)
1794
self.assertKnitFilesEqual(source, target)
1796
# The target knit object is in a consistent state, i.e. the record we
1797
# just added is immediately visible.
1798
self.assertTrue(target.has_version('text-a'))
1799
self.assertFalse(target.has_version('text-ghost'))
1800
self.assertEqual({'text-a':('text-ghost',)},
1801
target.get_parent_map(['text-a', 'text-ghost']))
1802
self.assertEqual(split_lines(TEXT_1), target.get_lines('text-a'))
1804
def test_inconsistent_version_lines(self):
1805
"""Inserting a data stream which has different content for a version_id
1806
than already exists in the knit will raise KnitCorrupt.
1808
source = self.make_test_knit(name='source')
1809
target = self.make_test_knit(name='target')
1810
# Insert a different 'text-a' into both source and target
1811
source.add_lines('text-a', [], split_lines(TEXT_1))
1812
target.add_lines('text-a', [], split_lines(TEXT_2))
1813
# Insert a data stream with conflicting content into the target
1814
data_stream = source.get_data_stream(['text-a'])
1816
errors.KnitCorrupt, target.insert_data_stream, data_stream)
1818
def test_inconsistent_version_parents(self):
1819
"""Inserting a data stream which has different parents for a version_id
1820
than already exists in the knit will raise KnitCorrupt.
1822
source = self.make_test_knit(name='source')
1823
target = self.make_test_knit(name='target')
1824
# Insert a different 'text-a' into both source and target. They differ
1825
# only by the parents list, the content is the same.
1826
source.add_lines_with_ghosts('text-a', [], split_lines(TEXT_1))
1827
target.add_lines_with_ghosts('text-a', ['a-ghost'], split_lines(TEXT_1))
1828
# Insert a data stream with conflicting content into the target
1829
data_stream = source.get_data_stream(['text-a'])
1831
errors.KnitCorrupt, target.insert_data_stream, data_stream)
1833
def test_unknown_stream_format(self):
1834
"""A data stream in a different format to the target knit cannot be
1837
It will raise KnitDataStreamUnknown because the fallback code will fail
1838
to make a knit. In future we may need KnitDataStreamIncompatible again,
1839
for more exotic cases.
1841
data_stream = ('fake-format-signature', [], lambda _: '')
1842
target = self.make_test_knit(name='target')
1844
errors.KnitDataStreamUnknown,
1845
target.insert_data_stream, data_stream)
1847
def test_bug_208418(self):
1848
"""You can insert a stream with an incompatible format, even when:
1849
* the stream has a line-delta record,
1850
* whose parent is in the target, also stored as a line-delta
1852
See <https://launchpad.net/bugs/208418>.
1854
base_lines = split_lines(TEXT_1)
1856
target = self.make_test_knit(name='target', annotate=True)
1857
target.add_lines('version-1', [], base_lines)
1858
target.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1859
# The second record should be a delta.
1860
self.assertEqual('line-delta', target._index.get_method('version-2'))
1862
# Make a source, with a different format, but the same data
1863
source = self.make_test_knit(name='source', annotate=False)
1864
source.add_lines('version-1', [], base_lines)
1865
source.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1866
# Now add another record, which should be stored as a delta against
1868
source.add_lines('version-3', ['version-2'], base_lines + ['b\n'])
1869
self.assertEqual('line-delta', source._index.get_method('version-3'))
1871
# Make a stream of the new version
1872
data_stream = source.get_data_stream(['version-3'])
1873
# And insert into the target
1874
target.insert_data_stream(data_stream)
1875
# No errors should have been raised.
1877
def test_line_delta_record_into_non_delta_knit(self):
1878
# Make a data stream with a line-delta record
1879
source = self.make_test_knit(name='source', delta=True)
1880
base_lines = split_lines(TEXT_1)
1881
source.add_lines('version-1', [], base_lines)
1882
source.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1883
# The second record should be a delta.
1884
self.assertEqual('line-delta', source._index.get_method('version-2'))
1885
data_stream = source.get_data_stream(['version-1', 'version-2'])
1887
# Insert the stream into a non-delta knit.
1888
target = self.make_test_knit(name='target', delta=False)
1889
target.insert_data_stream(data_stream)
1891
# Both versions are fulltexts in the target
1892
self.assertEqual('fulltext', target._index.get_method('version-1'))
1893
self.assertEqual('fulltext', target._index.get_method('version-2'))
1896
class DataStreamTests(KnitTests):
1898
def assertMadeStreamKnit(self, source_knit, versions, target_knit):
1899
"""Assert that a knit made from a stream is as expected."""
1900
a_stream = source_knit.get_data_stream(versions)
1901
expected_data = a_stream[2](None)
1902
a_stream = source_knit.get_data_stream(versions)
1903
a_knit = target_knit._knit_from_datastream(a_stream)
1904
self.assertEqual(source_knit.factory.__class__,
1905
a_knit.factory.__class__)
1906
self.assertIsInstance(a_knit._data._access, _StreamAccess)
1907
self.assertIsInstance(a_knit._index, _StreamIndex)
1908
self.assertEqual(a_knit._index.data_list, a_stream[1])
1909
self.assertEqual(a_knit._data._access.data, expected_data)
1910
self.assertEqual(a_knit.filename, target_knit.filename)
1911
self.assertEqual(a_knit.transport, target_knit.transport)
1912
self.assertEqual(a_knit._index, a_knit._data._access.stream_index)
1913
self.assertEqual(target_knit, a_knit._data._access.backing_knit)
1914
self.assertIsInstance(a_knit._data._access.orig_factory,
1915
source_knit.factory.__class__)
1917
def test__knit_from_data_stream_empty(self):
1918
"""Create a knit object from a datastream."""
1919
annotated = self.make_test_knit(name='source', annotate=True)
1920
plain = self.make_test_knit(name='target', annotate=False)
1921
# case 1: annotated source
1922
self.assertMadeStreamKnit(annotated, [], annotated)
1923
self.assertMadeStreamKnit(annotated, [], plain)
1924
# case 2: plain source
1925
self.assertMadeStreamKnit(plain, [], annotated)
1926
self.assertMadeStreamKnit(plain, [], plain)
1928
def test__knit_from_data_stream_unknown_format(self):
1929
annotated = self.make_test_knit(name='source', annotate=True)
1930
self.assertRaises(errors.KnitDataStreamUnknown,
1931
annotated._knit_from_datastream, ("unknown", None, None))
1943
Banana cup cake recipe
1949
- self-raising flour
1953
Banana cup cake recipe
1955
- bananas (do not use plantains!!!)
1962
Banana cup cake recipe
1965
- self-raising flour
1978
AB_MERGE_TEXT="""unchanged|Banana cup cake recipe
1983
new-b|- bananas (do not use plantains!!!)
1984
unchanged|- broken tea cups
1985
new-a|- self-raising flour
1988
AB_MERGE=[tuple(l.split('|')) for l in AB_MERGE_TEXT.splitlines(True)]
1991
def line_delta(from_lines, to_lines):
1992
"""Generate line-based delta from one text to another"""
1993
s = difflib.SequenceMatcher(None, from_lines, to_lines)
1994
for op in s.get_opcodes():
1995
if op[0] == 'equal':
1997
yield '%d,%d,%d\n' % (op[1], op[2], op[4]-op[3])
1998
for i in range(op[3], op[4]):
2002
def apply_line_delta(basis_lines, delta_lines):
2003
"""Apply a line-based perfect diff
2005
basis_lines -- text to apply the patch to
2006
delta_lines -- diff instructions and content
2008
out = basis_lines[:]
2011
while i < len(delta_lines):
2013
a, b, c = map(long, l.split(','))
2015
out[offset+a:offset+b] = delta_lines[i:i+c]
2017
offset = offset + (b - a) + c
2021
class TestWeaveToKnit(KnitTests):
2023
def test_weave_to_knit_matches(self):
2024
# check that the WeaveToKnit is_compatible function
2025
# registers True for a Weave to a Knit.
2026
w = Weave(get_scope=lambda:None)
2027
k = self.make_test_knit()
2028
self.failUnless(WeaveToKnit.is_compatible(w, k))
2029
self.failIf(WeaveToKnit.is_compatible(k, w))
2030
self.failIf(WeaveToKnit.is_compatible(w, w))
2031
self.failIf(WeaveToKnit.is_compatible(k, k))
1490
def make_test_knit(self, annotate=False, name='test'):
1491
mapper = ConstantMapper(name)
1492
return make_file_factory(annotate, mapper)(self.get_transport())
1495
class TestBadShaError(KnitTests):
1496
"""Tests for handling of sha errors."""
1498
def test_sha_exception_has_text(self):
1499
# having the failed text included in the error allows for recovery.
1500
source = self.make_test_knit()
1501
target = self.make_test_knit(name="target")
1502
if not source._max_delta_chain:
1503
raise TestNotApplicable(
1504
"cannot get delta-caused sha failures without deltas.")
1507
broken = ('broken',)
1508
source.add_lines(basis, (), ['foo\n'])
1509
source.add_lines(broken, (basis,), ['foo\n', 'bar\n'])
1510
# Seed target with a bad basis text
1511
target.add_lines(basis, (), ['gam\n'])
1512
target.insert_record_stream(
1513
source.get_record_stream([broken], 'unordered', False))
1514
err = self.assertRaises(errors.KnitCorrupt,
1515
target.get_record_stream([broken], 'unordered', True
1516
).next().get_bytes_as, 'chunked')
1517
self.assertEqual(['gam\n', 'bar\n'], err.content)
1518
# Test for formatting with live data
1519
self.assertStartsWith(str(err), "Knit ")
2034
1522
class TestKnitIndex(KnitTests):
2153
1650
add_callback = self.catch_add
2155
1652
add_callback = None
2156
return KnitGraphIndex(combined_index, deltas=deltas,
1653
return _KnitGraphIndex(combined_index, lambda:True, deltas=deltas,
2157
1654
add_callback=add_callback)
2159
def test_get_ancestry(self):
2160
# get_ancestry is defined as eliding ghosts, not erroring.
2161
index = self.two_graph_index()
2162
self.assertEqual([], index.get_ancestry([]))
2163
self.assertEqual(['separate'], index.get_ancestry(['separate']))
2164
self.assertEqual(['tail'], index.get_ancestry(['tail']))
2165
self.assertEqual(['tail', 'parent'], index.get_ancestry(['parent']))
2166
self.assertEqual(['tail', 'parent', 'tip'], index.get_ancestry(['tip']))
2167
self.assertTrue(index.get_ancestry(['tip', 'separate']) in
2168
(['tail', 'parent', 'tip', 'separate'],
2169
['separate', 'tail', 'parent', 'tip'],
2171
# and without topo_sort
2172
self.assertEqual(set(['separate']),
2173
set(index.get_ancestry(['separate'], topo_sorted=False)))
2174
self.assertEqual(set(['tail']),
2175
set(index.get_ancestry(['tail'], topo_sorted=False)))
2176
self.assertEqual(set(['tail', 'parent']),
2177
set(index.get_ancestry(['parent'], topo_sorted=False)))
2178
self.assertEqual(set(['tail', 'parent', 'tip']),
2179
set(index.get_ancestry(['tip'], topo_sorted=False)))
2180
self.assertEqual(set(['separate', 'tail', 'parent', 'tip']),
2181
set(index.get_ancestry(['tip', 'separate'])))
2182
# asking for a ghost makes it go boom.
2183
self.assertRaises(errors.RevisionNotPresent, index.get_ancestry, ['ghost'])
2185
def test_get_ancestry_with_ghosts(self):
2186
index = self.two_graph_index()
2187
self.assertEqual([], index.get_ancestry_with_ghosts([]))
2188
self.assertEqual(['separate'], index.get_ancestry_with_ghosts(['separate']))
2189
self.assertEqual(['tail'], index.get_ancestry_with_ghosts(['tail']))
2190
self.assertTrue(index.get_ancestry_with_ghosts(['parent']) in
2191
(['tail', 'ghost', 'parent'],
2192
['ghost', 'tail', 'parent'],
2194
self.assertTrue(index.get_ancestry_with_ghosts(['tip']) in
2195
(['tail', 'ghost', 'parent', 'tip'],
2196
['ghost', 'tail', 'parent', 'tip'],
2198
self.assertTrue(index.get_ancestry_with_ghosts(['tip', 'separate']) in
2199
(['tail', 'ghost', 'parent', 'tip', 'separate'],
2200
['ghost', 'tail', 'parent', 'tip', 'separate'],
2201
['separate', 'tail', 'ghost', 'parent', 'tip'],
2202
['separate', 'ghost', 'tail', 'parent', 'tip'],
2204
# asking for a ghost makes it go boom.
2205
self.assertRaises(errors.RevisionNotPresent, index.get_ancestry_with_ghosts, ['ghost'])
2207
def test_num_versions(self):
2208
index = self.two_graph_index()
2209
self.assertEqual(4, index.num_versions())
2211
def test_get_versions(self):
2212
index = self.two_graph_index()
2213
self.assertEqual(set(['tail', 'tip', 'parent', 'separate']),
2214
set(index.get_versions()))
2216
def test_has_version(self):
2217
index = self.two_graph_index()
2218
self.assertTrue(index.has_version('tail'))
2219
self.assertFalse(index.has_version('ghost'))
1656
def test_keys(self):
1657
index = self.two_graph_index()
1658
self.assertEqual(set([('tail',), ('tip',), ('parent',), ('separate',)]),
2221
1661
def test_get_position(self):
2222
1662
index = self.two_graph_index()
2223
self.assertEqual((index._graph_index._indices[0], 0, 100), index.get_position('tip'))
2224
self.assertEqual((index._graph_index._indices[1], 100, 78), index.get_position('parent'))
1663
self.assertEqual((index._graph_index._indices[0], 0, 100), index.get_position(('tip',)))
1664
self.assertEqual((index._graph_index._indices[1], 100, 78), index.get_position(('parent',)))
2226
1666
def test_get_method_deltas(self):
2227
1667
index = self.two_graph_index(deltas=True)
2228
self.assertEqual('fulltext', index.get_method('tip'))
2229
self.assertEqual('line-delta', index.get_method('parent'))
1668
self.assertEqual('fulltext', index.get_method(('tip',)))
1669
self.assertEqual('line-delta', index.get_method(('parent',)))
2231
1671
def test_get_method_no_deltas(self):
2232
1672
# check that the parent-history lookup is ignored with deltas=False.
2233
1673
index = self.two_graph_index(deltas=False)
2234
self.assertEqual('fulltext', index.get_method('tip'))
2235
self.assertEqual('fulltext', index.get_method('parent'))
1674
self.assertEqual('fulltext', index.get_method(('tip',)))
1675
self.assertEqual('fulltext', index.get_method(('parent',)))
2237
1677
def test_get_options_deltas(self):
2238
1678
index = self.two_graph_index(deltas=True)
2239
self.assertEqual(['fulltext', 'no-eol'], index.get_options('tip'))
2240
self.assertEqual(['line-delta'], index.get_options('parent'))
1679
self.assertEqual(['fulltext', 'no-eol'], index.get_options(('tip',)))
1680
self.assertEqual(['line-delta'], index.get_options(('parent',)))
2242
1682
def test_get_options_no_deltas(self):
2243
1683
# check that the parent-history lookup is ignored with deltas=False.
2244
1684
index = self.two_graph_index(deltas=False)
2245
self.assertEqual(['fulltext', 'no-eol'], index.get_options('tip'))
2246
self.assertEqual(['fulltext'], index.get_options('parent'))
2248
def test_get_parents_with_ghosts(self):
2249
index = self.two_graph_index()
2250
self.assertEqual(('tail', 'ghost'), index.get_parents_with_ghosts('parent'))
2251
# and errors on ghosts.
2252
self.assertRaises(errors.RevisionNotPresent,
2253
index.get_parents_with_ghosts, 'ghost')
2255
def test_check_versions_present(self):
2256
# ghosts should not be considered present
2257
index = self.two_graph_index()
2258
self.assertRaises(RevisionNotPresent, index.check_versions_present,
2260
self.assertRaises(RevisionNotPresent, index.check_versions_present,
2262
index.check_versions_present(['tail', 'separate'])
1685
self.assertEqual(['fulltext', 'no-eol'], index.get_options(('tip',)))
1686
self.assertEqual(['fulltext'], index.get_options(('parent',)))
1688
def test_get_parent_map(self):
1689
index = self.two_graph_index()
1690
self.assertEqual({('parent',):(('tail',), ('ghost',))},
1691
index.get_parent_map([('parent',), ('ghost',)]))
2264
1693
def catch_add(self, entries):
2265
1694
self.caught_entries.append(entries)
2267
1696
def test_add_no_callback_errors(self):
2268
1697
index = self.two_graph_index()
2269
self.assertRaises(errors.ReadOnlyError, index.add_version,
2270
'new', 'fulltext,no-eol', (None, 50, 60), ['separate'])
1698
self.assertRaises(errors.ReadOnlyError, index.add_records,
1699
[(('new',), 'fulltext,no-eol', (None, 50, 60), ['separate'])])
2272
1701
def test_add_version_smoke(self):
2273
1702
index = self.two_graph_index(catch_adds=True)
2274
index.add_version('new', 'fulltext,no-eol', (None, 50, 60), ['separate'])
1703
index.add_records([(('new',), 'fulltext,no-eol', (None, 50, 60),
2275
1705
self.assertEqual([[(('new', ), 'N50 60', ((('separate',),),))]],
2276
1706
self.caught_entries)
2278
1708
def test_add_version_delta_not_delta_index(self):
2279
1709
index = self.two_graph_index(catch_adds=True)
2280
self.assertRaises(errors.KnitCorrupt, index.add_version,
2281
'new', 'no-eol,line-delta', (None, 0, 100), ['parent'])
1710
self.assertRaises(errors.KnitCorrupt, index.add_records,
1711
[(('new',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
2282
1712
self.assertEqual([], self.caught_entries)
2284
1714
def test_add_version_same_dup(self):
2285
1715
index = self.two_graph_index(catch_adds=True)
2286
1716
# options can be spelt two different ways
2287
index.add_version('tip', 'fulltext,no-eol', (None, 0, 100), ['parent'])
2288
index.add_version('tip', 'no-eol,fulltext', (None, 0, 100), ['parent'])
2289
# but neither should have added data.
2290
self.assertEqual([[], []], self.caught_entries)
1717
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1718
index.add_records([(('tip',), 'no-eol,fulltext', (None, 0, 100), [('parent',)])])
1719
# position/length are ignored (because each pack could have fulltext or
1720
# delta, and be at a different position.
1721
index.add_records([(('tip',), 'fulltext,no-eol', (None, 50, 100),
1723
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000),
1725
# but neither should have added data:
1726
self.assertEqual([[], [], [], []], self.caught_entries)
2292
1728
def test_add_version_different_dup(self):
2293
1729
index = self.two_graph_index(deltas=True, catch_adds=True)
2294
1730
# change options
2295
self.assertRaises(errors.KnitCorrupt, index.add_version,
2296
'tip', 'no-eol,line-delta', (None, 0, 100), ['parent'])
2297
self.assertRaises(errors.KnitCorrupt, index.add_version,
2298
'tip', 'line-delta,no-eol', (None, 0, 100), ['parent'])
2299
self.assertRaises(errors.KnitCorrupt, index.add_version,
2300
'tip', 'fulltext', (None, 0, 100), ['parent'])
2302
self.assertRaises(errors.KnitCorrupt, index.add_version,
2303
'tip', 'fulltext,no-eol', (None, 50, 100), ['parent'])
2304
self.assertRaises(errors.KnitCorrupt, index.add_version,
2305
'tip', 'fulltext,no-eol', (None, 0, 1000), ['parent'])
1731
self.assertRaises(errors.KnitCorrupt, index.add_records,
1732
[(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1733
self.assertRaises(errors.KnitCorrupt, index.add_records,
1734
[(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
2307
self.assertRaises(errors.KnitCorrupt, index.add_version,
2308
'tip', 'fulltext,no-eol', (None, 0, 100), [])
1736
self.assertRaises(errors.KnitCorrupt, index.add_records,
1737
[(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
2309
1738
self.assertEqual([], self.caught_entries)
2311
1740
def test_add_versions_nodeltas(self):
2312
1741
index = self.two_graph_index(catch_adds=True)
2313
index.add_versions([
2314
('new', 'fulltext,no-eol', (None, 50, 60), ['separate']),
2315
('new2', 'fulltext', (None, 0, 6), ['new']),
1743
(('new',), 'fulltext,no-eol', (None, 50, 60), [('separate',)]),
1744
(('new2',), 'fulltext', (None, 0, 6), [('new',)]),
2317
1746
self.assertEqual([(('new', ), 'N50 60', ((('separate',),),)),
2318
1747
(('new2', ), ' 0 6', ((('new',),),))],
2544
2041
def test_add_versions_delta_not_delta_index(self):
2545
2042
index = self.two_graph_index(catch_adds=True)
2546
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2547
[('new', 'no-eol,line-delta', (None, 0, 100), ['parent'])])
2043
self.assertRaises(errors.KnitCorrupt, index.add_records,
2044
[(('new',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
2548
2045
self.assertEqual([], self.caught_entries)
2550
2047
def test_add_versions_parents_not_parents_index(self):
2551
2048
index = self.two_graph_index(catch_adds=True)
2552
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2553
[('new', 'no-eol,fulltext', (None, 0, 100), ['parent'])])
2049
self.assertRaises(errors.KnitCorrupt, index.add_records,
2050
[(('new',), 'no-eol,fulltext', (None, 0, 100), [('parent',)])])
2554
2051
self.assertEqual([], self.caught_entries)
2556
2053
def test_add_versions_random_id_accepted(self):
2557
2054
index = self.two_graph_index(catch_adds=True)
2558
index.add_versions([], random_id=True)
2055
index.add_records([], random_id=True)
2560
2057
def test_add_versions_same_dup(self):
2561
2058
index = self.two_graph_index(catch_adds=True)
2562
2059
# options can be spelt two different ways
2563
index.add_versions([('tip', 'fulltext,no-eol', (None, 0, 100), [])])
2564
index.add_versions([('tip', 'no-eol,fulltext', (None, 0, 100), [])])
2060
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
2061
index.add_records([(('tip',), 'no-eol,fulltext', (None, 0, 100), [])])
2062
# position/length are ignored (because each pack could have fulltext or
2063
# delta, and be at a different position.
2064
index.add_records([(('tip',), 'fulltext,no-eol', (None, 50, 100), [])])
2065
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
2565
2066
# but neither should have added data.
2566
self.assertEqual([[], []], self.caught_entries)
2067
self.assertEqual([[], [], [], []], self.caught_entries)
2568
2069
def test_add_versions_different_dup(self):
2569
2070
index = self.two_graph_index(catch_adds=True)
2570
2071
# change options
2571
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2572
[('tip', 'no-eol,line-delta', (None, 0, 100), [])])
2573
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2574
[('tip', 'line-delta,no-eol', (None, 0, 100), [])])
2575
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2576
[('tip', 'fulltext', (None, 0, 100), [])])
2578
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2579
[('tip', 'fulltext,no-eol', (None, 50, 100), [])])
2580
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2581
[('tip', 'fulltext,no-eol', (None, 0, 1000), [])])
2072
self.assertRaises(errors.KnitCorrupt, index.add_records,
2073
[(('tip',), 'no-eol,line-delta', (None, 0, 100), [])])
2074
self.assertRaises(errors.KnitCorrupt, index.add_records,
2075
[(('tip',), 'line-delta,no-eol', (None, 0, 100), [])])
2076
self.assertRaises(errors.KnitCorrupt, index.add_records,
2077
[(('tip',), 'fulltext', (None, 0, 100), [])])
2583
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2584
[('tip', 'fulltext,no-eol', (None, 0, 100), ['parent'])])
2079
self.assertRaises(errors.KnitCorrupt, index.add_records,
2080
[(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
2585
2081
# change options in the second record
2586
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2587
[('tip', 'fulltext,no-eol', (None, 0, 100), []),
2588
('tip', 'no-eol,line-delta', (None, 0, 100), [])])
2082
self.assertRaises(errors.KnitCorrupt, index.add_records,
2083
[(('tip',), 'fulltext,no-eol', (None, 0, 100), []),
2084
(('tip',), 'no-eol,line-delta', (None, 0, 100), [])])
2589
2085
self.assertEqual([], self.caught_entries)
2591
class TestPackKnits(KnitTests):
2592
"""Tests that use a _PackAccess and KnitGraphIndex."""
2594
def test_get_data_stream_packs_ignores_pack_overhead(self):
2595
# Packs have an encoding overhead that should not be included in the
2596
# 'size' field of a data stream, because it is not returned by the
2597
# raw_reading functions - it is why index_memo's are opaque, and
2598
# get_data_stream was abusing this.
2599
packname = 'test.pack'
2600
transport = self.get_transport()
2601
def write_data(bytes):
2602
transport.append_bytes(packname, bytes)
2603
writer = pack.ContainerWriter(write_data)
2605
index = InMemoryGraphIndex(2)
2606
knit_index = KnitGraphIndex(index, add_callback=index.add_nodes,
2608
indices = {index:(transport, packname)}
2609
access = _PackAccess(indices, writer=(writer, index))
2610
k = KnitVersionedFile('test', get_transport('.'),
2611
delta=True, create=True, index=knit_index, access_method=access)
2612
# insert something into the knit
2613
k.add_lines('text-1', [], ["foo\n"])
2614
# get a data stream for it
2615
stream = k.get_data_stream(['text-1'])
2616
# if the stream has been incorrectly assembled, we will get a short read
2617
# reading from the stream (as streams have no trailer)
2618
expected_length = stream[1][0][2]
2619
# we use -1 to do the read, so that if a trailer is added this test
2620
# will fail and we'll adjust it to handle that case correctly, rather
2621
# than allowing an over-read that is bogus.
2622
self.assertEqual(expected_length, len(stream[2](-1)))
2625
class Test_StreamIndex(KnitTests):
2627
def get_index(self, knit, stream):
2628
"""Get a _StreamIndex from knit and stream."""
2629
return knit._knit_from_datastream(stream)._index
2631
def assertIndexVersions(self, knit, versions):
2632
"""Check that the _StreamIndex versions are those of the stream."""
2633
index = self.get_index(knit, knit.get_data_stream(versions))
2634
self.assertEqual(set(index.get_versions()), set(versions))
2635
# check we didn't get duplicates
2636
self.assertEqual(len(index.get_versions()), len(versions))
2638
def assertIndexAncestry(self, knit, ancestry_versions, versions, result):
2639
"""Check the result of a get_ancestry call on knit."""
2640
index = self.get_index(knit, knit.get_data_stream(versions))
2643
set(index.get_ancestry(ancestry_versions, False)))
2645
def assertGetMethod(self, knit, versions, version, result):
2646
index = self.get_index(knit, knit.get_data_stream(versions))
2647
self.assertEqual(result, index.get_method(version))
2649
def assertGetOptions(self, knit, version, options):
2650
index = self.get_index(knit, knit.get_data_stream(version))
2651
self.assertEqual(options, index.get_options(version))
2653
def assertGetPosition(self, knit, versions, version, result):
2654
index = self.get_index(knit, knit.get_data_stream(versions))
2655
if result[1] is None:
2656
result = (result[0], index, result[2], result[3])
2657
self.assertEqual(result, index.get_position(version))
2659
def assertGetParentsWithGhosts(self, knit, versions, version, parents):
2660
index = self.get_index(knit, knit.get_data_stream(versions))
2661
self.assertEqual(parents, index.get_parents_with_ghosts(version))
2663
def make_knit_with_4_versions_2_dags(self):
2664
knit = self.make_test_knit()
2665
knit.add_lines('a', [], ["foo"])
2666
knit.add_lines('b', [], [])
2667
knit.add_lines('c', ['b', 'a'], [])
2668
knit.add_lines_with_ghosts('d', ['e', 'f'], [])
2671
def test_versions(self):
2672
"""The versions of a StreamIndex are those of the datastream."""
2673
knit = self.make_knit_with_4_versions_2_dags()
2674
# ask for most permutations, which catches bugs like falling back to the
2675
# target knit, or showing ghosts, etc.
2676
self.assertIndexVersions(knit, [])
2677
self.assertIndexVersions(knit, ['a'])
2678
self.assertIndexVersions(knit, ['b'])
2679
self.assertIndexVersions(knit, ['c'])
2680
self.assertIndexVersions(knit, ['d'])
2681
self.assertIndexVersions(knit, ['a', 'b'])
2682
self.assertIndexVersions(knit, ['b', 'c'])
2683
self.assertIndexVersions(knit, ['a', 'c'])
2684
self.assertIndexVersions(knit, ['a', 'b', 'c'])
2685
self.assertIndexVersions(knit, ['a', 'b', 'c', 'd'])
2687
def test_construct(self):
2688
"""Constructing a StreamIndex generates index data."""
2689
data_list = [('text-a', ['fulltext'], 127, []),
2690
('text-b', ['option'], 128, ['text-c'])]
2691
index = _StreamIndex(data_list, None)
2692
self.assertEqual({'text-a':(['fulltext'], (0, 127), []),
2693
'text-b':(['option'], (127, 127 + 128), ['text-c'])},
2696
def test_get_ancestry(self):
2697
knit = self.make_knit_with_4_versions_2_dags()
2698
self.assertIndexAncestry(knit, ['a'], ['a'], ['a'])
2699
self.assertIndexAncestry(knit, ['b'], ['b'], ['b'])
2700
self.assertIndexAncestry(knit, ['c'], ['c'], ['c'])
2701
self.assertIndexAncestry(knit, ['c'], ['a', 'b', 'c'],
2702
set(['a', 'b', 'c']))
2703
self.assertIndexAncestry(knit, ['c', 'd'], ['a', 'b', 'c', 'd'],
2704
set(['a', 'b', 'c', 'd']))
2706
def test_get_method(self):
2707
knit = self.make_knit_with_4_versions_2_dags()
2708
self.assertGetMethod(knit, ['a'], 'a', 'fulltext')
2709
self.assertGetMethod(knit, ['c'], 'c', 'line-delta')
2710
# get_method on a basis that is not in the datastream (but in the
2711
# backing knit) returns 'fulltext', because thats what we'll create as
2713
self.assertGetMethod(knit, ['c'], 'b', 'fulltext')
2715
def test_get_options(self):
2716
knit = self.make_knit_with_4_versions_2_dags()
2717
self.assertGetOptions(knit, 'a', ['no-eol', 'fulltext'])
2718
self.assertGetOptions(knit, 'c', ['line-delta'])
2720
def test_get_parents_with_ghosts(self):
2721
knit = self.make_knit_with_4_versions_2_dags()
2722
self.assertGetParentsWithGhosts(knit, ['a'], 'a', ())
2723
self.assertGetParentsWithGhosts(knit, ['c'], 'c', ('b', 'a'))
2724
self.assertGetParentsWithGhosts(knit, ['d'], 'd', ('e', 'f'))
2726
def test_get_position(self):
2727
knit = self.make_knit_with_4_versions_2_dags()
2728
# get_position returns (thunk_flag, index(can be None), start, end) for
2729
# _StreamAccess to use.
2730
self.assertGetPosition(knit, ['a'], 'a', (False, None, 0, 78))
2731
self.assertGetPosition(knit, ['a', 'c'], 'c', (False, None, 78, 156))
2732
# get_position on a text that is not in the datastream (but in the
2733
# backing knit) returns (True, 'versionid', None, None) - and then the
2734
# access object can construct the relevant data as needed.
2735
self.assertGetPosition(knit, ['a', 'c'], 'b', (True, 'b', None, None))
2738
class Test_StreamAccess(KnitTests):
2740
def get_index_access(self, knit, stream):
2741
"""Get a _StreamAccess from knit and stream."""
2742
knit = knit._knit_from_datastream(stream)
2743
return knit._index, knit._data._access
2745
def assertGetRawRecords(self, knit, versions):
2746
index, access = self.get_index_access(knit,
2747
knit.get_data_stream(versions))
2748
# check that every version asked for can be obtained from the resulting
2752
for version in versions:
2753
memos.append(knit._index.get_position(version))
2755
for version, data in zip(
2756
versions, knit._data._access.get_raw_records(memos)):
2757
original[version] = data
2759
for version in versions:
2760
memos.append(index.get_position(version))
2762
for version, data in zip(versions, access.get_raw_records(memos)):
2763
streamed[version] = data
2764
self.assertEqual(original, streamed)
2766
for version in versions:
2767
data = list(access.get_raw_records(
2768
[index.get_position(version)]))[0]
2769
self.assertEqual(original[version], data)
2771
def make_knit_with_two_versions(self):
2772
knit = self.make_test_knit()
2773
knit.add_lines('a', [], ["foo"])
2774
knit.add_lines('b', [], ["bar"])
2777
def test_get_raw_records(self):
2778
knit = self.make_knit_with_two_versions()
2779
self.assertGetRawRecords(knit, ['a', 'b'])
2780
self.assertGetRawRecords(knit, ['a'])
2781
self.assertGetRawRecords(knit, ['b'])
2783
def test_get_raw_record_from_backing_knit(self):
2784
# the thunk layer should create an artificial A on-demand when needed.
2785
source_knit = self.make_test_knit(name='plain', annotate=False)
2786
target_knit = self.make_test_knit(name='annotated', annotate=True)
2787
source_knit.add_lines("A", [], ["Foo\n"])
2788
# Give the target A, so we can try to thunk across to it.
2789
target_knit.insert_record_stream(source_knit.get_record_stream(['A'],
2088
class TestKnitVersionedFiles(KnitTests):
2090
def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
2091
positions, _min_buffer_size=None):
2092
kvf = self.make_test_knit()
2093
if _min_buffer_size is None:
2094
_min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
2095
self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
2096
non_local_keys, positions,
2097
_min_buffer_size=_min_buffer_size))
2099
def assertSplitByPrefix(self, expected_map, expected_prefix_order,
2101
split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
2102
self.assertEqual(expected_map, split)
2103
self.assertEqual(expected_prefix_order, prefix_order)
2105
def test__group_keys_for_io(self):
2106
ft_detail = ('fulltext', False)
2107
ld_detail = ('line-delta', False)
2115
f_a: (ft_detail, (f_a, 0, 100), None),
2116
f_b: (ld_detail, (f_b, 100, 21), f_a),
2117
f_c: (ld_detail, (f_c, 180, 15), f_b),
2118
g_a: (ft_detail, (g_a, 121, 35), None),
2119
g_b: (ld_detail, (g_b, 156, 12), g_a),
2120
g_c: (ld_detail, (g_c, 195, 13), g_a),
2122
self.assertGroupKeysForIo([([f_a], set())],
2123
[f_a], [], positions)
2124
self.assertGroupKeysForIo([([f_a], set([f_a]))],
2125
[f_a], [f_a], positions)
2126
self.assertGroupKeysForIo([([f_a, f_b], set([]))],
2127
[f_a, f_b], [], positions)
2128
self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
2129
[f_a, f_b], [f_b], positions)
2130
self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
2131
[f_a, g_a, f_b, g_b], [], positions)
2132
self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
2133
[f_a, g_a, f_b, g_b], [], positions,
2134
_min_buffer_size=150)
2135
self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
2136
[f_a, g_a, f_b, g_b], [], positions,
2137
_min_buffer_size=100)
2138
self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
2139
[f_c, g_b], [], positions,
2140
_min_buffer_size=125)
2141
self.assertGroupKeysForIo([([g_b, f_c], set())],
2142
[g_b, f_c], [], positions,
2143
_min_buffer_size=125)
2145
def test__split_by_prefix(self):
2146
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2147
'g': [('g', 'b'), ('g', 'a')],
2149
[('f', 'a'), ('g', 'b'),
2150
('g', 'a'), ('f', 'b')])
2152
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2153
'g': [('g', 'b'), ('g', 'a')],
2155
[('f', 'a'), ('f', 'b'),
2156
('g', 'b'), ('g', 'a')])
2158
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2159
'g': [('g', 'b'), ('g', 'a')],
2161
[('f', 'a'), ('f', 'b'),
2162
('g', 'b'), ('g', 'a')])
2164
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2165
'g': [('g', 'b'), ('g', 'a')],
2166
'': [('a',), ('b',)]
2168
[('f', 'a'), ('g', 'b'),
2170
('g', 'a'), ('f', 'b')])
2173
class TestStacking(KnitTests):
2175
def get_basis_and_test_knit(self):
2176
basis = self.make_test_knit(name='basis')
2177
basis = RecordingVersionedFilesDecorator(basis)
2178
test = self.make_test_knit(name='test')
2179
test.add_fallback_versioned_files(basis)
2182
def test_add_fallback_versioned_files(self):
2183
basis = self.make_test_knit(name='basis')
2184
test = self.make_test_knit(name='test')
2185
# It must not error; other tests test that the fallback is referred to
2186
# when accessing data.
2187
test.add_fallback_versioned_files(basis)
2189
def test_add_lines(self):
2190
# lines added to the test are not added to the basis
2191
basis, test = self.get_basis_and_test_knit()
2193
key_basis = ('bar',)
2194
key_cross_border = ('quux',)
2195
key_delta = ('zaphod',)
2196
test.add_lines(key, (), ['foo\n'])
2197
self.assertEqual({}, basis.get_parent_map([key]))
2198
# lines added to the test that reference across the stack do a
2200
basis.add_lines(key_basis, (), ['foo\n'])
2202
test.add_lines(key_cross_border, (key_basis,), ['foo\n'])
2203
self.assertEqual('fulltext', test._index.get_method(key_cross_border))
2204
# we don't even need to look at the basis to see that this should be
2205
# stored as a fulltext
2206
self.assertEqual([], basis.calls)
2207
# Subsequent adds do delta.
2209
test.add_lines(key_delta, (key_cross_border,), ['foo\n'])
2210
self.assertEqual('line-delta', test._index.get_method(key_delta))
2211
self.assertEqual([], basis.calls)
2213
def test_annotate(self):
2214
# annotations from the test knit are answered without asking the basis
2215
basis, test = self.get_basis_and_test_knit()
2217
key_basis = ('bar',)
2218
key_missing = ('missing',)
2219
test.add_lines(key, (), ['foo\n'])
2220
details = test.annotate(key)
2221
self.assertEqual([(key, 'foo\n')], details)
2222
self.assertEqual([], basis.calls)
2223
# But texts that are not in the test knit are looked for in the basis
2225
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2227
details = test.annotate(key_basis)
2228
self.assertEqual([(key_basis, 'foo\n'), (key_basis, 'bar\n')], details)
2229
# Not optimised to date:
2230
# self.assertEqual([("annotate", key_basis)], basis.calls)
2231
self.assertEqual([('get_parent_map', set([key_basis])),
2232
('get_parent_map', set([key_basis])),
2233
('get_record_stream', [key_basis], 'topological', True)],
2236
def test_check(self):
2237
# At the moment checking a stacked knit does implicitly check the
2239
basis, test = self.get_basis_and_test_knit()
2242
def test_get_parent_map(self):
2243
# parents in the test knit are answered without asking the basis
2244
basis, test = self.get_basis_and_test_knit()
2246
key_basis = ('bar',)
2247
key_missing = ('missing',)
2248
test.add_lines(key, (), [])
2249
parent_map = test.get_parent_map([key])
2250
self.assertEqual({key: ()}, parent_map)
2251
self.assertEqual([], basis.calls)
2252
# But parents that are not in the test knit are looked for in the basis
2253
basis.add_lines(key_basis, (), [])
2255
parent_map = test.get_parent_map([key, key_basis, key_missing])
2256
self.assertEqual({key: (),
2257
key_basis: ()}, parent_map)
2258
self.assertEqual([("get_parent_map", set([key_basis, key_missing]))],
2261
def test_get_record_stream_unordered_fulltexts(self):
2262
# records from the test knit are answered without asking the basis:
2263
basis, test = self.get_basis_and_test_knit()
2265
key_basis = ('bar',)
2266
key_missing = ('missing',)
2267
test.add_lines(key, (), ['foo\n'])
2268
records = list(test.get_record_stream([key], 'unordered', True))
2269
self.assertEqual(1, len(records))
2270
self.assertEqual([], basis.calls)
2271
# Missing (from test knit) objects are retrieved from the basis:
2272
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2274
records = list(test.get_record_stream([key_basis, key_missing],
2276
self.assertEqual(2, len(records))
2277
calls = list(basis.calls)
2278
for record in records:
2279
self.assertSubset([record.key], (key_basis, key_missing))
2280
if record.key == key_missing:
2281
self.assertIsInstance(record, AbsentContentFactory)
2283
reference = list(basis.get_record_stream([key_basis],
2284
'unordered', True))[0]
2285
self.assertEqual(reference.key, record.key)
2286
self.assertEqual(reference.sha1, record.sha1)
2287
self.assertEqual(reference.storage_kind, record.storage_kind)
2288
self.assertEqual(reference.get_bytes_as(reference.storage_kind),
2289
record.get_bytes_as(record.storage_kind))
2290
self.assertEqual(reference.get_bytes_as('fulltext'),
2291
record.get_bytes_as('fulltext'))
2292
# It's not strictly minimal, but it seems reasonable for now for it to
2293
# ask which fallbacks have which parents.
2295
("get_parent_map", set([key_basis, key_missing])),
2296
("get_record_stream", [key_basis], 'unordered', True)],
2299
def test_get_record_stream_ordered_fulltexts(self):
2300
# ordering is preserved down into the fallback store.
2301
basis, test = self.get_basis_and_test_knit()
2303
key_basis = ('bar',)
2304
key_basis_2 = ('quux',)
2305
key_missing = ('missing',)
2306
test.add_lines(key, (key_basis,), ['foo\n'])
2307
# Missing (from test knit) objects are retrieved from the basis:
2308
basis.add_lines(key_basis, (key_basis_2,), ['foo\n', 'bar\n'])
2309
basis.add_lines(key_basis_2, (), ['quux\n'])
2311
# ask for in non-topological order
2312
records = list(test.get_record_stream(
2313
[key, key_basis, key_missing, key_basis_2], 'topological', True))
2314
self.assertEqual(4, len(records))
2316
for record in records:
2317
self.assertSubset([record.key],
2318
(key_basis, key_missing, key_basis_2, key))
2319
if record.key == key_missing:
2320
self.assertIsInstance(record, AbsentContentFactory)
2322
results.append((record.key, record.sha1, record.storage_kind,
2323
record.get_bytes_as('fulltext')))
2324
calls = list(basis.calls)
2325
order = [record[0] for record in results]
2326
self.assertEqual([key_basis_2, key_basis, key], order)
2327
for result in results:
2328
if result[0] == key:
2332
record = source.get_record_stream([result[0]], 'unordered',
2334
self.assertEqual(record.key, result[0])
2335
self.assertEqual(record.sha1, result[1])
2336
# We used to check that the storage kind matched, but actually it
2337
# depends on whether it was sourced from the basis, or in a single
2338
# group, because asking for full texts returns proxy objects to a
2339
# _ContentMapGenerator object; so checking the kind is unneeded.
2340
self.assertEqual(record.get_bytes_as('fulltext'), result[3])
2341
# It's not strictly minimal, but it seems reasonable for now for it to
2342
# ask which fallbacks have which parents.
2344
("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2345
# topological is requested from the fallback, because that is what
2346
# was requested at the top level.
2347
("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
2350
def test_get_record_stream_unordered_deltas(self):
2351
# records from the test knit are answered without asking the basis:
2352
basis, test = self.get_basis_and_test_knit()
2354
key_basis = ('bar',)
2355
key_missing = ('missing',)
2356
test.add_lines(key, (), ['foo\n'])
2357
records = list(test.get_record_stream([key], 'unordered', False))
2358
self.assertEqual(1, len(records))
2359
self.assertEqual([], basis.calls)
2360
# Missing (from test knit) objects are retrieved from the basis:
2361
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2363
records = list(test.get_record_stream([key_basis, key_missing],
2790
2364
'unordered', False))
2791
index, access = self.get_index_access(target_knit,
2792
source_knit.get_data_stream([]))
2793
raw_data = list(access.get_raw_records([(True, "A", None, None)]))[0]
2794
df = GzipFile(mode='rb', fileobj=StringIO(raw_data))
2796
'version A 1 5d36b88bb697a2d778f024048bafabd443d74503\n'
2800
def test_asking_for_thunk_stream_is_not_plain_errors(self):
2801
knit = self.make_test_knit(name='annotated', annotate=True)
2802
knit.add_lines("A", [], ["Foo\n"])
2803
index, access = self.get_index_access(knit,
2804
knit.get_data_stream([]))
2805
self.assertRaises(errors.KnitCorrupt,
2806
list, access.get_raw_records([(True, "A", None, None)]))
2809
class TestFormatSignatures(KnitTests):
2811
def test_knit_format_signatures(self):
2812
"""Different formats of knit have different signature strings."""
2813
knit = self.make_test_knit(name='a', annotate=True)
2814
self.assertEqual('knit-annotated', knit.get_format_signature())
2815
knit = self.make_test_knit(name='p', annotate=False)
2816
self.assertEqual('knit-plain', knit.get_format_signature())
2365
self.assertEqual(2, len(records))
2366
calls = list(basis.calls)
2367
for record in records:
2368
self.assertSubset([record.key], (key_basis, key_missing))
2369
if record.key == key_missing:
2370
self.assertIsInstance(record, AbsentContentFactory)
2372
reference = list(basis.get_record_stream([key_basis],
2373
'unordered', False))[0]
2374
self.assertEqual(reference.key, record.key)
2375
self.assertEqual(reference.sha1, record.sha1)
2376
self.assertEqual(reference.storage_kind, record.storage_kind)
2377
self.assertEqual(reference.get_bytes_as(reference.storage_kind),
2378
record.get_bytes_as(record.storage_kind))
2379
# It's not strictly minimal, but it seems reasonable for now for it to
2380
# ask which fallbacks have which parents.
2382
("get_parent_map", set([key_basis, key_missing])),
2383
("get_record_stream", [key_basis], 'unordered', False)],
2386
def test_get_record_stream_ordered_deltas(self):
2387
# ordering is preserved down into the fallback store.
2388
basis, test = self.get_basis_and_test_knit()
2390
key_basis = ('bar',)
2391
key_basis_2 = ('quux',)
2392
key_missing = ('missing',)
2393
test.add_lines(key, (key_basis,), ['foo\n'])
2394
# Missing (from test knit) objects are retrieved from the basis:
2395
basis.add_lines(key_basis, (key_basis_2,), ['foo\n', 'bar\n'])
2396
basis.add_lines(key_basis_2, (), ['quux\n'])
2398
# ask for in non-topological order
2399
records = list(test.get_record_stream(
2400
[key, key_basis, key_missing, key_basis_2], 'topological', False))
2401
self.assertEqual(4, len(records))
2403
for record in records:
2404
self.assertSubset([record.key],
2405
(key_basis, key_missing, key_basis_2, key))
2406
if record.key == key_missing:
2407
self.assertIsInstance(record, AbsentContentFactory)
2409
results.append((record.key, record.sha1, record.storage_kind,
2410
record.get_bytes_as(record.storage_kind)))
2411
calls = list(basis.calls)
2412
order = [record[0] for record in results]
2413
self.assertEqual([key_basis_2, key_basis, key], order)
2414
for result in results:
2415
if result[0] == key:
2419
record = source.get_record_stream([result[0]], 'unordered',
2421
self.assertEqual(record.key, result[0])
2422
self.assertEqual(record.sha1, result[1])
2423
self.assertEqual(record.storage_kind, result[2])
2424
self.assertEqual(record.get_bytes_as(record.storage_kind), result[3])
2425
# It's not strictly minimal, but it seems reasonable for now for it to
2426
# ask which fallbacks have which parents.
2428
("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2429
("get_record_stream", [key_basis_2, key_basis], 'topological', False)],
2432
def test_get_sha1s(self):
2433
# sha1's in the test knit are answered without asking the basis
2434
basis, test = self.get_basis_and_test_knit()
2436
key_basis = ('bar',)
2437
key_missing = ('missing',)
2438
test.add_lines(key, (), ['foo\n'])
2439
key_sha1sum = osutils.sha('foo\n').hexdigest()
2440
sha1s = test.get_sha1s([key])
2441
self.assertEqual({key: key_sha1sum}, sha1s)
2442
self.assertEqual([], basis.calls)
2443
# But texts that are not in the test knit are looked for in the basis
2444
# directly (rather than via text reconstruction) so that remote servers
2445
# etc don't have to answer with full content.
2446
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2447
basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2449
sha1s = test.get_sha1s([key, key_missing, key_basis])
2450
self.assertEqual({key: key_sha1sum,
2451
key_basis: basis_sha1sum}, sha1s)
2452
self.assertEqual([("get_sha1s", set([key_basis, key_missing]))],
2455
def test_insert_record_stream(self):
2456
# records are inserted as normal; insert_record_stream builds on
2457
# add_lines, so a smoke test should be all that's needed:
2459
key_basis = ('bar',)
2460
key_delta = ('zaphod',)
2461
basis, test = self.get_basis_and_test_knit()
2462
source = self.make_test_knit(name='source')
2463
basis.add_lines(key_basis, (), ['foo\n'])
2465
source.add_lines(key_basis, (), ['foo\n'])
2466
source.add_lines(key_delta, (key_basis,), ['bar\n'])
2467
stream = source.get_record_stream([key_delta], 'unordered', False)
2468
test.insert_record_stream(stream)
2469
# XXX: this does somewhat too many calls in making sure of whether it
2470
# has to recreate the full text.
2471
self.assertEqual([("get_parent_map", set([key_basis])),
2472
('get_parent_map', set([key_basis])),
2473
('get_record_stream', [key_basis], 'unordered', True)],
2475
self.assertEqual({key_delta:(key_basis,)},
2476
test.get_parent_map([key_delta]))
2477
self.assertEqual('bar\n', test.get_record_stream([key_delta],
2478
'unordered', True).next().get_bytes_as('fulltext'))
2480
def test_iter_lines_added_or_present_in_keys(self):
2481
# Lines from the basis are returned, and lines for a given key are only
2485
# all sources are asked for keys:
2486
basis, test = self.get_basis_and_test_knit()
2487
basis.add_lines(key1, (), ["foo"])
2489
lines = list(test.iter_lines_added_or_present_in_keys([key1]))
2490
self.assertEqual([("foo\n", key1)], lines)
2491
self.assertEqual([("iter_lines_added_or_present_in_keys", set([key1]))],
2493
# keys in both are not duplicated:
2494
test.add_lines(key2, (), ["bar\n"])
2495
basis.add_lines(key2, (), ["bar\n"])
2497
lines = list(test.iter_lines_added_or_present_in_keys([key2]))
2498
self.assertEqual([("bar\n", key2)], lines)
2499
self.assertEqual([], basis.calls)
2501
def test_keys(self):
2504
# all sources are asked for keys:
2505
basis, test = self.get_basis_and_test_knit()
2507
self.assertEqual(set(), set(keys))
2508
self.assertEqual([("keys",)], basis.calls)
2509
# keys from a basis are returned:
2510
basis.add_lines(key1, (), [])
2513
self.assertEqual(set([key1]), set(keys))
2514
self.assertEqual([("keys",)], basis.calls)
2515
# keys in both are not duplicated:
2516
test.add_lines(key2, (), [])
2517
basis.add_lines(key2, (), [])
2520
self.assertEqual(2, len(keys))
2521
self.assertEqual(set([key1, key2]), set(keys))
2522
self.assertEqual([("keys",)], basis.calls)
2524
def test_add_mpdiffs(self):
2525
# records are inserted as normal; add_mpdiff builds on
2526
# add_lines, so a smoke test should be all that's needed:
2528
key_basis = ('bar',)
2529
key_delta = ('zaphod',)
2530
basis, test = self.get_basis_and_test_knit()
2531
source = self.make_test_knit(name='source')
2532
basis.add_lines(key_basis, (), ['foo\n'])
2534
source.add_lines(key_basis, (), ['foo\n'])
2535
source.add_lines(key_delta, (key_basis,), ['bar\n'])
2536
diffs = source.make_mpdiffs([key_delta])
2537
test.add_mpdiffs([(key_delta, (key_basis,),
2538
source.get_sha1s([key_delta])[key_delta], diffs[0])])
2539
self.assertEqual([("get_parent_map", set([key_basis])),
2540
('get_record_stream', [key_basis], 'unordered', True),],
2542
self.assertEqual({key_delta:(key_basis,)},
2543
test.get_parent_map([key_delta]))
2544
self.assertEqual('bar\n', test.get_record_stream([key_delta],
2545
'unordered', True).next().get_bytes_as('fulltext'))
2547
def test_make_mpdiffs(self):
2548
# Generating an mpdiff across a stacking boundary should detect parent
2552
key_right = ('zaphod',)
2553
basis, test = self.get_basis_and_test_knit()
2554
basis.add_lines(key_left, (), ['bar\n'])
2555
basis.add_lines(key_right, (), ['zaphod\n'])
2557
test.add_lines(key, (key_left, key_right),
2558
['bar\n', 'foo\n', 'zaphod\n'])
2559
diffs = test.make_mpdiffs([key])
2561
multiparent.MultiParent([multiparent.ParentText(0, 0, 0, 1),
2562
multiparent.NewText(['foo\n']),
2563
multiparent.ParentText(1, 0, 2, 1)])],
2565
self.assertEqual(3, len(basis.calls))
2567
("get_parent_map", set([key_left, key_right])),
2568
("get_parent_map", set([key_left, key_right])),
2571
last_call = basis.calls[-1]
2572
self.assertEqual('get_record_stream', last_call[0])
2573
self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2574
self.assertEqual('topological', last_call[2])
2575
self.assertEqual(True, last_call[3])
2578
class TestNetworkBehaviour(KnitTests):
2579
"""Tests for getting data out of/into knits over the network."""
2581
def test_include_delta_closure_generates_a_knit_delta_closure(self):
2582
vf = self.make_test_knit(name='test')
2583
# put in three texts, giving ft, delta, delta
2584
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2585
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2586
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2587
# But heuristics could interfere, so check what happened:
2588
self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
2589
[record.storage_kind for record in
2590
vf.get_record_stream([('base',), ('d1',), ('d2',)],
2591
'topological', False)])
2592
# generate a stream of just the deltas include_delta_closure=True,
2593
# serialise to the network, and check that we get a delta closure on the wire.
2594
stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
2595
netb = [record.get_bytes_as(record.storage_kind) for record in stream]
2596
# The first bytes should be a memo from _ContentMapGenerator, and the
2597
# second bytes should be empty (because its a API proxy not something
2598
# for wire serialisation.
2599
self.assertEqual('', netb[1])
2601
kind, line_end = network_bytes_to_kind_and_offset(bytes)
2602
self.assertEqual('knit-delta-closure', kind)
2605
class TestContentMapGenerator(KnitTests):
2606
"""Tests for ContentMapGenerator"""
2608
def test_get_record_stream_gives_records(self):
2609
vf = self.make_test_knit(name='test')
2610
# put in three texts, giving ft, delta, delta
2611
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2612
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2613
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2614
keys = [('d1',), ('d2',)]
2615
generator = _VFContentMapGenerator(vf, keys,
2616
global_map=vf.get_parent_map(keys))
2617
for record in generator.get_record_stream():
2618
if record.key == ('d1',):
2619
self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
2621
self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
2623
def test_get_record_stream_kinds_are_raw(self):
2624
vf = self.make_test_knit(name='test')
2625
# put in three texts, giving ft, delta, delta
2626
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2627
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2628
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2629
keys = [('base',), ('d1',), ('d2',)]
2630
generator = _VFContentMapGenerator(vf, keys,
2631
global_map=vf.get_parent_map(keys))
2632
kinds = {('base',): 'knit-delta-closure',
2633
('d1',): 'knit-delta-closure-ref',
2634
('d2',): 'knit-delta-closure-ref',
2636
for record in generator.get_record_stream():
2637
self.assertEqual(kinds[record.key], record.storage_kind)
added
removed
bzrlib/tests/test_knit.py
