462
340
writer = pack.ContainerWriter(write_data)
464
342
access.set_writer(writer, index, (transport, packname))
465
memos = access.add_raw_records([('key', 10)], '1234567890')
343
memos = access.add_raw_records([10], '1234567890')
467
345
self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
469
def test_missing_index_raises_retry(self):
470
memos = self.make_pack_file()
471
transport = self.get_transport()
472
reload_called, reload_func = self.make_reload_func()
473
# Note that the index key has changed from 'foo' to 'bar'
474
access = _DirectPackAccess({'bar':(transport, 'packname')},
475
reload_func=reload_func)
476
e = self.assertListRaises(errors.RetryWithNewPacks,
477
access.get_raw_records, memos)
478
# Because a key was passed in which does not match our index list, we
479
# assume that the listing was already reloaded
480
self.assertTrue(e.reload_occurred)
481
self.assertIsInstance(e.exc_info, tuple)
482
self.assertIs(e.exc_info[0], KeyError)
483
self.assertIsInstance(e.exc_info[1], KeyError)
485
def test_missing_index_raises_key_error_with_no_reload(self):
486
memos = self.make_pack_file()
487
transport = self.get_transport()
488
# Note that the index key has changed from 'foo' to 'bar'
489
access = _DirectPackAccess({'bar':(transport, 'packname')})
490
e = self.assertListRaises(KeyError, access.get_raw_records, memos)
492
def test_missing_file_raises_retry(self):
493
memos = self.make_pack_file()
494
transport = self.get_transport()
495
reload_called, reload_func = self.make_reload_func()
496
# Note that the 'filename' has been changed to 'different-packname'
497
access = _DirectPackAccess({'foo':(transport, 'different-packname')},
498
reload_func=reload_func)
499
e = self.assertListRaises(errors.RetryWithNewPacks,
500
access.get_raw_records, memos)
501
# The file has gone missing, so we assume we need to reload
502
self.assertFalse(e.reload_occurred)
503
self.assertIsInstance(e.exc_info, tuple)
504
self.assertIs(e.exc_info[0], errors.NoSuchFile)
505
self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
506
self.assertEqual('different-packname', e.exc_info[1].path)
508
def test_missing_file_raises_no_such_file_with_no_reload(self):
509
memos = self.make_pack_file()
510
transport = self.get_transport()
511
# Note that the 'filename' has been changed to 'different-packname'
512
access = _DirectPackAccess({'foo':(transport, 'different-packname')})
513
e = self.assertListRaises(errors.NoSuchFile,
514
access.get_raw_records, memos)
516
def test_failing_readv_raises_retry(self):
517
memos = self.make_pack_file()
518
transport = self.get_transport()
519
failing_transport = MockReadvFailingTransport(
520
[transport.get_bytes('packname')])
521
reload_called, reload_func = self.make_reload_func()
522
access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
523
reload_func=reload_func)
524
# Asking for a single record will not trigger the Mock failure
525
self.assertEqual(['1234567890'],
526
list(access.get_raw_records(memos[:1])))
527
self.assertEqual(['12345'],
528
list(access.get_raw_records(memos[1:2])))
529
# A multiple offset readv() will fail mid-way through
530
e = self.assertListRaises(errors.RetryWithNewPacks,
531
access.get_raw_records, memos)
532
# The file has gone missing, so we assume we need to reload
533
self.assertFalse(e.reload_occurred)
534
self.assertIsInstance(e.exc_info, tuple)
535
self.assertIs(e.exc_info[0], errors.NoSuchFile)
536
self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
537
self.assertEqual('packname', e.exc_info[1].path)
539
def test_failing_readv_raises_no_such_file_with_no_reload(self):
540
memos = self.make_pack_file()
541
transport = self.get_transport()
542
failing_transport = MockReadvFailingTransport(
543
[transport.get_bytes('packname')])
544
reload_called, reload_func = self.make_reload_func()
545
access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
546
# Asking for a single record will not trigger the Mock failure
547
self.assertEqual(['1234567890'],
548
list(access.get_raw_records(memos[:1])))
549
self.assertEqual(['12345'],
550
list(access.get_raw_records(memos[1:2])))
551
# A multiple offset readv() will fail mid-way through
552
e = self.assertListRaises(errors.NoSuchFile,
553
access.get_raw_records, memos)
555
def test_reload_or_raise_no_reload(self):
556
access = _DirectPackAccess({}, reload_func=None)
557
retry_exc = self.make_retry_exception()
558
# Without a reload_func, we will just re-raise the original exception
559
self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
561
def test_reload_or_raise_reload_changed(self):
562
reload_called, reload_func = self.make_reload_func(return_val=True)
563
access = _DirectPackAccess({}, reload_func=reload_func)
564
retry_exc = self.make_retry_exception()
565
access.reload_or_raise(retry_exc)
566
self.assertEqual([1], reload_called)
567
retry_exc.reload_occurred=True
568
access.reload_or_raise(retry_exc)
569
self.assertEqual([2], reload_called)
571
def test_reload_or_raise_reload_no_change(self):
572
reload_called, reload_func = self.make_reload_func(return_val=False)
573
access = _DirectPackAccess({}, reload_func=reload_func)
574
retry_exc = self.make_retry_exception()
575
# If reload_occurred is False, then we consider it an error to have
576
# reload_func() return False (no changes).
577
self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
578
self.assertEqual([1], reload_called)
579
retry_exc.reload_occurred=True
580
# If reload_occurred is True, then we assume nothing changed because
581
# it had changed earlier, but didn't change again
582
access.reload_or_raise(retry_exc)
583
self.assertEqual([2], reload_called)
585
def test_annotate_retries(self):
586
vf, reload_counter = self.make_vf_for_retrying()
587
# It is a little bit bogus to annotate the Revision VF, but it works,
588
# as we have ancestry stored there
590
reload_lines = vf.annotate(key)
591
self.assertEqual([1, 1, 0], reload_counter)
592
plain_lines = vf.annotate(key)
593
self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
594
if reload_lines != plain_lines:
595
self.fail('Annotation was not identical with reloading.')
596
# Now delete the packs-in-use, which should trigger another reload, but
597
# this time we just raise an exception because we can't recover
598
for trans, name in vf._access._indices.itervalues():
600
self.assertRaises(errors.NoSuchFile, vf.annotate, key)
601
self.assertEqual([2, 1, 1], reload_counter)
603
def test__get_record_map_retries(self):
604
vf, reload_counter = self.make_vf_for_retrying()
605
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
606
records = vf._get_record_map(keys)
607
self.assertEqual(keys, sorted(records.keys()))
608
self.assertEqual([1, 1, 0], reload_counter)
609
# Now delete the packs-in-use, which should trigger another reload, but
610
# this time we just raise an exception because we can't recover
611
for trans, name in vf._access._indices.itervalues():
613
self.assertRaises(errors.NoSuchFile, vf._get_record_map, keys)
614
self.assertEqual([2, 1, 1], reload_counter)
616
def test_get_record_stream_retries(self):
617
vf, reload_counter = self.make_vf_for_retrying()
618
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
619
record_stream = vf.get_record_stream(keys, 'topological', False)
620
record = record_stream.next()
621
self.assertEqual(('rev-1',), record.key)
622
self.assertEqual([0, 0, 0], reload_counter)
623
record = record_stream.next()
624
self.assertEqual(('rev-2',), record.key)
625
self.assertEqual([1, 1, 0], reload_counter)
626
record = record_stream.next()
627
self.assertEqual(('rev-3',), record.key)
628
self.assertEqual([1, 1, 0], reload_counter)
629
# Now delete all pack files, and see that we raise the right error
630
for trans, name in vf._access._indices.itervalues():
632
self.assertListRaises(errors.NoSuchFile,
633
vf.get_record_stream, keys, 'topological', False)
635
def test_iter_lines_added_or_present_in_keys_retries(self):
636
vf, reload_counter = self.make_vf_for_retrying()
637
keys = [('rev-1',), ('rev-2',), ('rev-3',)]
638
# Unfortunately, iter_lines_added_or_present_in_keys iterates the
639
# result in random order (determined by the iteration order from a
640
# set()), so we don't have any solid way to trigger whether data is
641
# read before or after. However we tried to delete the middle node to
642
# exercise the code well.
643
# What we care about is that all lines are always yielded, but not
646
reload_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
647
self.assertEqual([1, 1, 0], reload_counter)
648
# Now do it again, to make sure the result is equivalent
649
plain_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
650
self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
651
self.assertEqual(plain_lines, reload_lines)
652
self.assertEqual(21, len(plain_lines))
653
# Now delete all pack files, and see that we raise the right error
654
for trans, name in vf._access._indices.itervalues():
656
self.assertListRaises(errors.NoSuchFile,
657
vf.iter_lines_added_or_present_in_keys, keys)
658
self.assertEqual([2, 1, 1], reload_counter)
660
def test_get_record_stream_yields_disk_sorted_order(self):
661
# if we get 'unordered' pick a semi-optimal order for reading. The
662
# order should be grouped by pack file, and then by position in file
663
repo = self.make_repository('test', format='pack-0.92')
665
self.addCleanup(repo.unlock)
666
repo.start_write_group()
668
vf.add_lines(('f-id', 'rev-5'), [('f-id', 'rev-4')], ['lines\n'])
669
vf.add_lines(('f-id', 'rev-1'), [], ['lines\n'])
670
vf.add_lines(('f-id', 'rev-2'), [('f-id', 'rev-1')], ['lines\n'])
671
repo.commit_write_group()
672
# We inserted them as rev-5, rev-1, rev-2, we should get them back in
674
stream = vf.get_record_stream([('f-id', 'rev-1'), ('f-id', 'rev-5'),
675
('f-id', 'rev-2')], 'unordered', False)
676
keys = [r.key for r in stream]
677
self.assertEqual([('f-id', 'rev-5'), ('f-id', 'rev-1'),
678
('f-id', 'rev-2')], keys)
679
repo.start_write_group()
680
vf.add_lines(('f-id', 'rev-4'), [('f-id', 'rev-3')], ['lines\n'])
681
vf.add_lines(('f-id', 'rev-3'), [('f-id', 'rev-2')], ['lines\n'])
682
vf.add_lines(('f-id', 'rev-6'), [('f-id', 'rev-5')], ['lines\n'])
683
repo.commit_write_group()
684
# Request in random order, to make sure the output order isn't based on
686
request_keys = set(('f-id', 'rev-%d' % i) for i in range(1, 7))
687
stream = vf.get_record_stream(request_keys, 'unordered', False)
688
keys = [r.key for r in stream]
689
# We want to get the keys back in disk order, but it doesn't matter
690
# which pack we read from first. So this can come back in 2 orders
691
alt1 = [('f-id', 'rev-%d' % i) for i in [4, 3, 6, 5, 1, 2]]
692
alt2 = [('f-id', 'rev-%d' % i) for i in [5, 1, 2, 4, 3, 6]]
693
if keys != alt1 and keys != alt2:
694
self.fail('Returned key order did not match either expected order.'
695
' expected %s or %s, not %s'
696
% (alt1, alt2, keys))
699
348
class LowLevelKnitDataTests(TestCase):
848
454
# Change 2 bytes in the middle to \xff
849
455
gz_txt = gz_txt[:10] + '\xff\xff' + gz_txt[12:]
850
456
transport = MockTransport([gz_txt])
851
access = _KnitKeyAccess(transport, ConstantMapper('filename'))
852
knit = KnitVersionedFiles(None, access)
853
records = [(('rev-id-1',), (('rev-id-1',), 0, len(gz_txt)))]
854
self.assertRaises(errors.KnitCorrupt, list,
855
knit._read_records_iter(records))
856
# read_records_iter_raw will barf on bad gz data
857
self.assertRaises(errors.KnitCorrupt, list,
858
knit._read_records_iter_raw(records))
457
access = _KnitAccess(transport, 'filename', None, None, False, False)
458
data = _KnitData(access=access)
459
records = [('rev-id-1', (None, 0, len(gz_txt)))]
461
self.assertRaises(errors.KnitCorrupt, data.read_records, records)
463
# read_records_iter_raw will notice if we request the wrong version.
464
self.assertRaises(errors.KnitCorrupt, list,
465
data.read_records_iter_raw(records))
861
468
class LowLevelKnitIndexTests(TestCase):
863
def get_knit_index(self, transport, name, mode):
864
mapper = ConstantMapper(name)
470
def get_knit_index(self, *args, **kwargs):
471
orig = knit._load_data
473
knit._load_data = orig
474
self.addCleanup(reset)
865
475
from bzrlib._knit_load_data_py import _load_data_py
866
self.overrideAttr(knit, '_load_data', _load_data_py)
867
allow_writes = lambda: 'w' in mode
868
return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
476
knit._load_data = _load_data_py
477
return _KnitIndex(get_scope=lambda:None, *args, **kwargs)
479
def test_no_such_file(self):
480
transport = MockTransport()
482
self.assertRaises(NoSuchFile, self.get_knit_index,
483
transport, "filename", "r")
484
self.assertRaises(NoSuchFile, self.get_knit_index,
485
transport, "filename", "w", create=False)
870
487
def test_create_file(self):
871
488
transport = MockTransport()
872
index = self.get_knit_index(transport, "filename", "w")
874
call = transport.calls.pop(0)
875
# call[1][1] is a StringIO - we can't test it by simple equality.
876
self.assertEqual('put_file_non_atomic', call[0])
877
self.assertEqual('filename.kndx', call[1][0])
878
# With no history, _KndxIndex writes a new index:
879
self.assertEqual(_KndxIndex.HEADER,
880
call[1][1].getvalue())
881
self.assertEqual({'create_parent_dir': True}, call[2])
490
index = self.get_knit_index(transport, "filename", "w",
491
file_mode="wb", create=True)
493
("put_bytes_non_atomic",
494
("filename", index.HEADER), {"mode": "wb"}),
495
transport.calls.pop(0))
497
def test_delay_create_file(self):
498
transport = MockTransport()
500
index = self.get_knit_index(transport, "filename", "w",
501
create=True, file_mode="wb", create_parent_dir=True,
502
delay_create=True, dir_mode=0777)
503
self.assertEqual([], transport.calls)
505
index.add_versions([])
506
name, (filename, f), kwargs = transport.calls.pop(0)
507
self.assertEqual("put_file_non_atomic", name)
509
{"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
511
self.assertEqual("filename", filename)
512
self.assertEqual(index.HEADER, f.read())
514
index.add_versions([])
515
self.assertEqual(("append_bytes", ("filename", ""), {}),
516
transport.calls.pop(0))
883
518
def test_read_utf8_version_id(self):
884
519
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
885
520
utf8_revision_id = unicode_revision_id.encode('utf-8')
886
521
transport = MockTransport([
888
523
'%s option 0 1 :' % (utf8_revision_id,)
890
525
index = self.get_knit_index(transport, "filename", "r")
891
# _KndxIndex is a private class, and deals in utf8 revision_ids, not
526
# _KnitIndex is a private class, and deals in utf8 revision_ids, not
892
527
# Unicode revision_ids.
893
self.assertEqual({(utf8_revision_id,):()},
894
index.get_parent_map(index.keys()))
895
self.assertFalse((unicode_revision_id,) in index.keys())
528
self.assertTrue(index.has_version(utf8_revision_id))
529
self.assertFalse(index.has_version(unicode_revision_id))
897
531
def test_read_utf8_parents(self):
898
532
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
899
533
utf8_revision_id = unicode_revision_id.encode('utf-8')
900
534
transport = MockTransport([
902
536
"version option 0 1 .%s :" % (utf8_revision_id,)
904
538
index = self.get_knit_index(transport, "filename", "r")
905
self.assertEqual({("version",):((utf8_revision_id,),)},
906
index.get_parent_map(index.keys()))
539
self.assertEqual((utf8_revision_id,),
540
index.get_parents_with_ghosts("version"))
908
542
def test_read_ignore_corrupted_lines(self):
909
543
transport = MockTransport([
912
546
"corrupted options 0 1 .b .c ",
913
547
"version options 0 1 :"
915
549
index = self.get_knit_index(transport, "filename", "r")
916
self.assertEqual(1, len(index.keys()))
917
self.assertEqual(set([("version",)]), index.keys())
550
self.assertEqual(1, index.num_versions())
551
self.assertTrue(index.has_version("version"))
919
553
def test_read_corrupted_header(self):
920
554
transport = MockTransport(['not a bzr knit index header\n'])
921
index = self.get_knit_index(transport, "filename", "r")
922
self.assertRaises(KnitHeaderError, index.keys)
555
self.assertRaises(KnitHeaderError,
556
self.get_knit_index, transport, "filename", "r")
924
558
def test_read_duplicate_entries(self):
925
559
transport = MockTransport([
927
561
"parent options 0 1 :",
928
562
"version options1 0 1 0 :",
929
563
"version options2 1 2 .other :",
930
564
"version options3 3 4 0 .other :"
932
566
index = self.get_knit_index(transport, "filename", "r")
933
self.assertEqual(2, len(index.keys()))
567
self.assertEqual(2, index.num_versions())
934
568
# check that the index used is the first one written. (Specific
935
569
# to KnitIndex style indices.
936
self.assertEqual("1", index._dictionary_compress([("version",)]))
937
self.assertEqual((("version",), 3, 4), index.get_position(("version",)))
938
self.assertEqual(["options3"], index.get_options(("version",)))
939
self.assertEqual({("version",):(("parent",), ("other",))},
940
index.get_parent_map([("version",)]))
570
self.assertEqual("1", index._version_list_to_index(["version"]))
571
self.assertEqual((None, 3, 4), index.get_position("version"))
572
self.assertEqual(["options3"], index.get_options("version"))
573
self.assertEqual(("parent", "other"),
574
index.get_parents_with_ghosts("version"))
942
576
def test_read_compressed_parents(self):
943
577
transport = MockTransport([
945
579
"a option 0 1 :",
946
580
"b option 0 1 0 :",
947
581
"c option 0 1 1 0 :",
949
583
index = self.get_knit_index(transport, "filename", "r")
950
self.assertEqual({("b",):(("a",),), ("c",):(("b",), ("a",))},
951
index.get_parent_map([("b",), ("c",)]))
584
self.assertEqual({"b":("a",), "c":("b", "a")},
585
index.get_parent_map(["b", "c"]))
953
587
def test_write_utf8_version_id(self):
954
588
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
955
589
utf8_revision_id = unicode_revision_id.encode('utf-8')
956
590
transport = MockTransport([
959
593
index = self.get_knit_index(transport, "filename", "r")
961
((utf8_revision_id,), ["option"], ((utf8_revision_id,), 0, 1), [])])
962
call = transport.calls.pop(0)
963
# call[1][1] is a StringIO - we can't test it by simple equality.
964
self.assertEqual('put_file_non_atomic', call[0])
965
self.assertEqual('filename.kndx', call[1][0])
966
# With no history, _KndxIndex writes a new index:
967
self.assertEqual(_KndxIndex.HEADER +
968
"\n%s option 0 1 :" % (utf8_revision_id,),
969
call[1][1].getvalue())
970
self.assertEqual({'create_parent_dir': True}, call[2])
594
index.add_version(utf8_revision_id, ["option"], (None, 0, 1), [])
595
self.assertEqual(("append_bytes", ("filename",
596
"\n%s option 0 1 :" % (utf8_revision_id,)),
598
transport.calls.pop(0))
972
600
def test_write_utf8_parents(self):
973
601
unicode_revision_id = u"version-\N{CYRILLIC CAPITAL LETTER A}"
974
602
utf8_revision_id = unicode_revision_id.encode('utf-8')
975
603
transport = MockTransport([
978
index = self.get_knit_index(transport, "filename", "r")
980
(("version",), ["option"], (("version",), 0, 1), [(utf8_revision_id,)])])
981
call = transport.calls.pop(0)
982
# call[1][1] is a StringIO - we can't test it by simple equality.
983
self.assertEqual('put_file_non_atomic', call[0])
984
self.assertEqual('filename.kndx', call[1][0])
985
# With no history, _KndxIndex writes a new index:
986
self.assertEqual(_KndxIndex.HEADER +
987
"\nversion option 0 1 .%s :" % (utf8_revision_id,),
988
call[1][1].getvalue())
989
self.assertEqual({'create_parent_dir': True}, call[2])
992
transport = MockTransport([
995
index = self.get_knit_index(transport, "filename", "r")
997
self.assertEqual(set(), index.keys())
999
index.add_records([(("a",), ["option"], (("a",), 0, 1), [])])
1000
self.assertEqual(set([("a",)]), index.keys())
1002
index.add_records([(("a",), ["option"], (("a",), 0, 1), [])])
1003
self.assertEqual(set([("a",)]), index.keys())
1005
index.add_records([(("b",), ["option"], (("b",), 0, 1), [])])
1006
self.assertEqual(set([("a",), ("b",)]), index.keys())
1008
def add_a_b(self, index, random_id=None):
1010
if random_id is not None:
1011
kwargs["random_id"] = random_id
1013
(("a",), ["option"], (("a",), 0, 1), [("b",)]),
1014
(("a",), ["opt"], (("a",), 1, 2), [("c",)]),
1015
(("b",), ["option"], (("b",), 2, 3), [("a",)])
1018
def assertIndexIsAB(self, index):
1023
index.get_parent_map(index.keys()))
1024
self.assertEqual((("a",), 1, 2), index.get_position(("a",)))
1025
self.assertEqual((("b",), 2, 3), index.get_position(("b",)))
1026
self.assertEqual(["opt"], index.get_options(("a",)))
606
index = self.get_knit_index(transport, "filename", "r")
607
index.add_version("version", ["option"], (None, 0, 1), [utf8_revision_id])
608
self.assertEqual(("append_bytes", ("filename",
609
"\nversion option 0 1 .%s :" % (utf8_revision_id,)),
611
transport.calls.pop(0))
613
def test_get_ancestry(self):
614
transport = MockTransport([
617
"b option 0 1 0 .e :",
618
"c option 0 1 1 0 :",
619
"d option 0 1 2 .f :"
621
index = self.get_knit_index(transport, "filename", "r")
623
self.assertEqual([], index.get_ancestry([]))
624
self.assertEqual(["a"], index.get_ancestry(["a"]))
625
self.assertEqual(["a", "b"], index.get_ancestry(["b"]))
626
self.assertEqual(["a", "b", "c"], index.get_ancestry(["c"]))
627
self.assertEqual(["a", "b", "c", "d"], index.get_ancestry(["d"]))
628
self.assertEqual(["a", "b"], index.get_ancestry(["a", "b"]))
629
self.assertEqual(["a", "b", "c"], index.get_ancestry(["a", "c"]))
631
self.assertRaises(RevisionNotPresent, index.get_ancestry, ["e"])
633
def test_get_ancestry_with_ghosts(self):
634
transport = MockTransport([
637
"b option 0 1 0 .e :",
638
"c option 0 1 0 .f .g :",
639
"d option 0 1 2 .h .j .k :"
641
index = self.get_knit_index(transport, "filename", "r")
643
self.assertEqual([], index.get_ancestry_with_ghosts([]))
644
self.assertEqual(["a"], index.get_ancestry_with_ghosts(["a"]))
645
self.assertEqual(["a", "e", "b"],
646
index.get_ancestry_with_ghosts(["b"]))
647
self.assertEqual(["a", "g", "f", "c"],
648
index.get_ancestry_with_ghosts(["c"]))
649
self.assertEqual(["a", "g", "f", "c", "k", "j", "h", "d"],
650
index.get_ancestry_with_ghosts(["d"]))
651
self.assertEqual(["a", "e", "b"],
652
index.get_ancestry_with_ghosts(["a", "b"]))
653
self.assertEqual(["a", "g", "f", "c"],
654
index.get_ancestry_with_ghosts(["a", "c"]))
656
["a", "g", "f", "c", "e", "b", "k", "j", "h", "d"],
657
index.get_ancestry_with_ghosts(["b", "d"]))
659
self.assertRaises(RevisionNotPresent,
660
index.get_ancestry_with_ghosts, ["e"])
662
def test_num_versions(self):
663
transport = MockTransport([
666
index = self.get_knit_index(transport, "filename", "r")
668
self.assertEqual(0, index.num_versions())
669
self.assertEqual(0, len(index))
671
index.add_version("a", ["option"], (None, 0, 1), [])
672
self.assertEqual(1, index.num_versions())
673
self.assertEqual(1, len(index))
675
index.add_version("a", ["option2"], (None, 1, 2), [])
676
self.assertEqual(1, index.num_versions())
677
self.assertEqual(1, len(index))
679
index.add_version("b", ["option"], (None, 0, 1), [])
680
self.assertEqual(2, index.num_versions())
681
self.assertEqual(2, len(index))
683
def test_get_versions(self):
684
transport = MockTransport([
687
index = self.get_knit_index(transport, "filename", "r")
689
self.assertEqual([], index.get_versions())
691
index.add_version("a", ["option"], (None, 0, 1), [])
692
self.assertEqual(["a"], index.get_versions())
694
index.add_version("a", ["option"], (None, 0, 1), [])
695
self.assertEqual(["a"], index.get_versions())
697
index.add_version("b", ["option"], (None, 0, 1), [])
698
self.assertEqual(["a", "b"], index.get_versions())
700
def test_add_version(self):
701
transport = MockTransport([
704
index = self.get_knit_index(transport, "filename", "r")
706
index.add_version("a", ["option"], (None, 0, 1), ["b"])
707
self.assertEqual(("append_bytes",
708
("filename", "\na option 0 1 .b :"),
709
{}), transport.calls.pop(0))
710
self.assertTrue(index.has_version("a"))
711
self.assertEqual(1, index.num_versions())
712
self.assertEqual((None, 0, 1), index.get_position("a"))
713
self.assertEqual(["option"], index.get_options("a"))
714
self.assertEqual(("b",), index.get_parents_with_ghosts("a"))
716
index.add_version("a", ["opt"], (None, 1, 2), ["c"])
717
self.assertEqual(("append_bytes",
718
("filename", "\na opt 1 2 .c :"),
719
{}), transport.calls.pop(0))
720
self.assertTrue(index.has_version("a"))
721
self.assertEqual(1, index.num_versions())
722
self.assertEqual((None, 1, 2), index.get_position("a"))
723
self.assertEqual(["opt"], index.get_options("a"))
724
self.assertEqual(("c",), index.get_parents_with_ghosts("a"))
726
index.add_version("b", ["option"], (None, 2, 3), ["a"])
727
self.assertEqual(("append_bytes",
728
("filename", "\nb option 2 3 0 :"),
729
{}), transport.calls.pop(0))
730
self.assertTrue(index.has_version("b"))
731
self.assertEqual(2, index.num_versions())
732
self.assertEqual((None, 2, 3), index.get_position("b"))
733
self.assertEqual(["option"], index.get_options("b"))
734
self.assertEqual(("a",), index.get_parents_with_ghosts("b"))
1028
736
def test_add_versions(self):
1029
737
transport = MockTransport([
1032
740
index = self.get_knit_index(transport, "filename", "r")
1035
call = transport.calls.pop(0)
1036
# call[1][1] is a StringIO - we can't test it by simple equality.
1037
self.assertEqual('put_file_non_atomic', call[0])
1038
self.assertEqual('filename.kndx', call[1][0])
1039
# With no history, _KndxIndex writes a new index:
743
("a", ["option"], (None, 0, 1), ["b"]),
744
("a", ["opt"], (None, 1, 2), ["c"]),
745
("b", ["option"], (None, 2, 3), ["a"])
747
self.assertEqual(("append_bytes", ("filename",
1042
748
"\na option 0 1 .b :"
1043
749
"\na opt 1 2 .c :"
1044
"\nb option 2 3 0 :",
1045
call[1][1].getvalue())
1046
self.assertEqual({'create_parent_dir': True}, call[2])
1047
self.assertIndexIsAB(index)
751
), {}), transport.calls.pop(0))
752
self.assertTrue(index.has_version("a"))
753
self.assertTrue(index.has_version("b"))
754
self.assertEqual(2, index.num_versions())
755
self.assertEqual((None, 1, 2), index.get_position("a"))
756
self.assertEqual((None, 2, 3), index.get_position("b"))
757
self.assertEqual(["opt"], index.get_options("a"))
758
self.assertEqual(["option"], index.get_options("b"))
759
self.assertEqual(("c",), index.get_parents_with_ghosts("a"))
760
self.assertEqual(("a",), index.get_parents_with_ghosts("b"))
1049
762
def test_add_versions_random_id_is_accepted(self):
1050
763
transport = MockTransport([
1053
766
index = self.get_knit_index(transport, "filename", "r")
1054
self.add_a_b(index, random_id=True)
769
("a", ["option"], (None, 0, 1), ["b"]),
770
("a", ["opt"], (None, 1, 2), ["c"]),
771
("b", ["option"], (None, 2, 3), ["a"])
1056
774
def test_delay_create_and_add_versions(self):
1057
775
transport = MockTransport()
1059
index = self.get_knit_index(transport, "filename", "w")
777
index = self.get_knit_index(transport, "filename", "w",
778
create=True, file_mode="wb", create_parent_dir=True,
779
delay_create=True, dir_mode=0777)
1061
780
self.assertEqual([], transport.calls)
1064
#[ {"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
1066
# Two calls: one during which we load the existing index (and when its
1067
# missing create it), then a second where we write the contents out.
1068
self.assertEqual(2, len(transport.calls))
1069
call = transport.calls.pop(0)
1070
self.assertEqual('put_file_non_atomic', call[0])
1071
self.assertEqual('filename.kndx', call[1][0])
1072
# With no history, _KndxIndex writes a new index:
1073
self.assertEqual(_KndxIndex.HEADER, call[1][1].getvalue())
1074
self.assertEqual({'create_parent_dir': True}, call[2])
1075
call = transport.calls.pop(0)
1076
# call[1][1] is a StringIO - we can't test it by simple equality.
1077
self.assertEqual('put_file_non_atomic', call[0])
1078
self.assertEqual('filename.kndx', call[1][0])
1079
# With no history, _KndxIndex writes a new index:
783
("a", ["option"], (None, 0, 1), ["b"]),
784
("a", ["opt"], (None, 1, 2), ["c"]),
785
("b", ["option"], (None, 2, 3), ["a"])
787
name, (filename, f), kwargs = transport.calls.pop(0)
788
self.assertEqual("put_file_non_atomic", name)
790
{"dir_mode": 0777, "create_parent_dir": True, "mode": "wb"},
792
self.assertEqual("filename", filename)
1082
795
"\na option 0 1 .b :"
1083
796
"\na opt 1 2 .c :"
1084
797
"\nb option 2 3 0 :",
1085
call[1][1].getvalue())
1086
self.assertEqual({'create_parent_dir': True}, call[2])
1088
def assertTotalBuildSize(self, size, keys, positions):
1089
self.assertEqual(size,
1090
knit._get_total_build_size(None, keys, positions))
1092
def test__get_total_build_size(self):
1094
('a',): (('fulltext', False), (('a',), 0, 100), None),
1095
('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
1096
('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
1097
('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
1099
self.assertTotalBuildSize(100, [('a',)], positions)
1100
self.assertTotalBuildSize(121, [('b',)], positions)
1101
# c needs both a & b
1102
self.assertTotalBuildSize(156, [('c',)], positions)
1103
# we shouldn't count 'b' twice
1104
self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
1105
self.assertTotalBuildSize(133, [('d',)], positions)
1106
self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
800
def test_has_version(self):
801
transport = MockTransport([
805
index = self.get_knit_index(transport, "filename", "r")
807
self.assertTrue(index.has_version("a"))
808
self.assertFalse(index.has_version("b"))
1108
810
def test_get_position(self):
1109
811
transport = MockTransport([
1111
813
"a option 0 1 :",
1112
814
"b option 1 2 :"
1114
816
index = self.get_knit_index(transport, "filename", "r")
1116
self.assertEqual((("a",), 0, 1), index.get_position(("a",)))
1117
self.assertEqual((("b",), 1, 2), index.get_position(("b",)))
818
self.assertEqual((None, 0, 1), index.get_position("a"))
819
self.assertEqual((None, 1, 2), index.get_position("b"))
1119
821
def test_get_method(self):
1120
822
transport = MockTransport([
1122
824
"a fulltext,unknown 0 1 :",
1123
825
"b unknown,line-delta 1 2 :",
1254
def test_scan_unvalidated_index_not_implemented(self):
1255
transport = MockTransport()
1256
index = self.get_knit_index(transport, 'filename', 'r')
1258
NotImplementedError, index.scan_unvalidated_index,
1259
'dummy graph_index')
1261
NotImplementedError, index.get_missing_compression_parents)
1263
987
def test_short_line(self):
1264
988
transport = MockTransport([
1266
990
"a option 0 10 :",
1267
991
"b option 10 10 0", # This line isn't terminated, ignored
1269
993
index = self.get_knit_index(transport, "filename", "r")
1270
self.assertEqual(set([('a',)]), index.keys())
994
self.assertEqual(['a'], index.get_versions())
1272
996
def test_skip_incomplete_record(self):
1273
997
# A line with bogus data should just be skipped
1274
998
transport = MockTransport([
1276
1000
"a option 0 10 :",
1277
1001
"b option 10 10 0", # This line isn't terminated, ignored
1278
1002
"c option 20 10 0 :", # Properly terminated, and starts with '\n'
1280
1004
index = self.get_knit_index(transport, "filename", "r")
1281
self.assertEqual(set([('a',), ('c',)]), index.keys())
1005
self.assertEqual(['a', 'c'], index.get_versions())
1283
1007
def test_trailing_characters(self):
1284
1008
# A line with bogus data should just be skipped
1285
1009
transport = MockTransport([
1287
1011
"a option 0 10 :",
1288
1012
"b option 10 10 0 :a", # This line has extra trailing characters
1289
1013
"c option 20 10 0 :", # Properly terminated, and starts with '\n'
1291
1015
index = self.get_knit_index(transport, "filename", "r")
1292
self.assertEqual(set([('a',), ('c',)]), index.keys())
1016
self.assertEqual(['a', 'c'], index.get_versions())
1295
1019
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1297
_test_needs_features = [compiled_knit_feature]
1299
def get_knit_index(self, transport, name, mode):
1300
mapper = ConstantMapper(name)
1301
from bzrlib._knit_load_data_pyx import _load_data_c
1302
self.overrideAttr(knit, '_load_data', _load_data_c)
1303
allow_writes = lambda: mode == 'w'
1304
return _KndxIndex(transport, mapper, lambda:None,
1305
allow_writes, lambda:True)
1308
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1310
def make_annotator(self):
1311
factory = knit.make_pack_factory(True, True, 1)
1312
vf = factory(self.get_transport())
1313
return knit._KnitAnnotator(vf)
1315
def test__expand_fulltext(self):
1316
ann = self.make_annotator()
1317
rev_key = ('rev-id',)
1318
ann._num_compression_children[rev_key] = 1
1319
res = ann._expand_record(rev_key, (('parent-id',),), None,
1320
['line1\n', 'line2\n'], ('fulltext', True))
1321
# The content object and text lines should be cached appropriately
1322
self.assertEqual(['line1\n', 'line2'], res)
1323
content_obj = ann._content_objects[rev_key]
1324
self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
1325
self.assertEqual(res, content_obj.text())
1326
self.assertEqual(res, ann._text_cache[rev_key])
1328
def test__expand_delta_comp_parent_not_available(self):
1329
# Parent isn't available yet, so we return nothing, but queue up this
1330
# node for later processing
1331
ann = self.make_annotator()
1332
rev_key = ('rev-id',)
1333
parent_key = ('parent-id',)
1334
record = ['0,1,1\n', 'new-line\n']
1335
details = ('line-delta', False)
1336
res = ann._expand_record(rev_key, (parent_key,), parent_key,
1338
self.assertEqual(None, res)
1339
self.assertTrue(parent_key in ann._pending_deltas)
1340
pending = ann._pending_deltas[parent_key]
1341
self.assertEqual(1, len(pending))
1342
self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
1344
def test__expand_record_tracks_num_children(self):
1345
ann = self.make_annotator()
1346
rev_key = ('rev-id',)
1347
rev2_key = ('rev2-id',)
1348
parent_key = ('parent-id',)
1349
record = ['0,1,1\n', 'new-line\n']
1350
details = ('line-delta', False)
1351
ann._num_compression_children[parent_key] = 2
1352
ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
1353
('fulltext', False))
1354
res = ann._expand_record(rev_key, (parent_key,), parent_key,
1356
self.assertEqual({parent_key: 1}, ann._num_compression_children)
1357
# Expanding the second child should remove the content object, and the
1358
# num_compression_children entry
1359
res = ann._expand_record(rev2_key, (parent_key,), parent_key,
1361
self.assertFalse(parent_key in ann._content_objects)
1362
self.assertEqual({}, ann._num_compression_children)
1363
# We should not cache the content_objects for rev2 and rev, because
1364
# they do not have compression children of their own.
1365
self.assertEqual({}, ann._content_objects)
1367
def test__expand_delta_records_blocks(self):
1368
ann = self.make_annotator()
1369
rev_key = ('rev-id',)
1370
parent_key = ('parent-id',)
1371
record = ['0,1,1\n', 'new-line\n']
1372
details = ('line-delta', True)
1373
ann._num_compression_children[parent_key] = 2
1374
ann._expand_record(parent_key, (), None,
1375
['line1\n', 'line2\n', 'line3\n'],
1376
('fulltext', False))
1377
ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
1378
self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
1379
ann._matching_blocks)
1380
rev2_key = ('rev2-id',)
1381
record = ['0,1,1\n', 'new-line\n']
1382
details = ('line-delta', False)
1383
ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
1384
self.assertEqual([(1, 1, 2), (3, 3, 0)],
1385
ann._matching_blocks[(rev2_key, parent_key)])
1387
def test__get_parent_ann_uses_matching_blocks(self):
1388
ann = self.make_annotator()
1389
rev_key = ('rev-id',)
1390
parent_key = ('parent-id',)
1391
parent_ann = [(parent_key,)]*3
1392
block_key = (rev_key, parent_key)
1393
ann._annotations_cache[parent_key] = parent_ann
1394
ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
1395
# We should not try to access any parent_lines content, because we know
1396
# we already have the matching blocks
1397
par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
1398
['1\n', '2\n', '3\n'], parent_key)
1399
self.assertEqual(parent_ann, par_ann)
1400
self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
1401
self.assertEqual({}, ann._matching_blocks)
1403
def test__process_pending(self):
1404
ann = self.make_annotator()
1405
rev_key = ('rev-id',)
1408
record = ['0,1,1\n', 'new-line\n']
1409
details = ('line-delta', False)
1410
p1_record = ['line1\n', 'line2\n']
1411
ann._num_compression_children[p1_key] = 1
1412
res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
1414
self.assertEqual(None, res)
1415
# self.assertTrue(p1_key in ann._pending_deltas)
1416
self.assertEqual({}, ann._pending_annotation)
1417
# Now insert p1, and we should be able to expand the delta
1418
res = ann._expand_record(p1_key, (), None, p1_record,
1419
('fulltext', False))
1420
self.assertEqual(p1_record, res)
1421
ann._annotations_cache[p1_key] = [(p1_key,)]*2
1422
res = ann._process_pending(p1_key)
1423
self.assertEqual([], res)
1424
self.assertFalse(p1_key in ann._pending_deltas)
1425
self.assertTrue(p2_key in ann._pending_annotation)
1426
self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
1427
ann._pending_annotation)
1428
# Now fill in parent 2, and pending annotation should be satisfied
1429
res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
1430
ann._annotations_cache[p2_key] = []
1431
res = ann._process_pending(p2_key)
1432
self.assertEqual([rev_key], res)
1433
self.assertEqual({}, ann._pending_annotation)
1434
self.assertEqual({}, ann._pending_deltas)
1436
def test_record_delta_removes_basis(self):
1437
ann = self.make_annotator()
1438
ann._expand_record(('parent-id',), (), None,
1439
['line1\n', 'line2\n'], ('fulltext', False))
1440
ann._num_compression_children['parent-id'] = 2
1442
def test_annotate_special_text(self):
1443
ann = self.make_annotator()
1445
rev1_key = ('rev-1',)
1446
rev2_key = ('rev-2',)
1447
rev3_key = ('rev-3',)
1448
spec_key = ('special:',)
1449
vf.add_lines(rev1_key, [], ['initial content\n'])
1450
vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
1453
vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
1456
spec_text = ('initial content\n'
1460
ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
1461
anns, lines = ann.annotate(spec_key)
1462
self.assertEqual([(rev1_key,),
1463
(rev2_key, rev3_key),
1467
self.assertEqualDiff(spec_text, ''.join(lines))
1021
_test_needs_features = [CompiledKnitFeature]
1023
def get_knit_index(self, *args, **kwargs):
1024
orig = knit._load_data
1026
knit._load_data = orig
1027
self.addCleanup(reset)
1028
from bzrlib._knit_load_data_c import _load_data_c
1029
knit._load_data = _load_data_c
1030
return _KnitIndex(get_scope=lambda:None, *args, **kwargs)
1470
1033
class KnitTests(TestCaseWithTransport):
1471
1034
"""Class containing knit test helper routines."""
1473
def make_test_knit(self, annotate=False, name='test'):
1474
mapper = ConstantMapper(name)
1475
return make_file_factory(annotate, mapper)(self.get_transport())
1478
class TestBadShaError(KnitTests):
1479
"""Tests for handling of sha errors."""
1481
def test_sha_exception_has_text(self):
1482
# having the failed text included in the error allows for recovery.
1483
source = self.make_test_knit()
1484
target = self.make_test_knit(name="target")
1485
if not source._max_delta_chain:
1486
raise TestNotApplicable(
1487
"cannot get delta-caused sha failures without deltas.")
1490
broken = ('broken',)
1491
source.add_lines(basis, (), ['foo\n'])
1492
source.add_lines(broken, (basis,), ['foo\n', 'bar\n'])
1493
# Seed target with a bad basis text
1494
target.add_lines(basis, (), ['gam\n'])
1495
target.insert_record_stream(
1496
source.get_record_stream([broken], 'unordered', False))
1497
err = self.assertRaises(errors.KnitCorrupt,
1498
target.get_record_stream([broken], 'unordered', True
1499
).next().get_bytes_as, 'chunked')
1500
self.assertEqual(['gam\n', 'bar\n'], err.content)
1501
# Test for formatting with live data
1502
self.assertStartsWith(str(err), "Knit ")
1036
def make_test_knit(self, annotate=False, delay_create=False, index=None,
1037
name='test', delta=True, access_mode='w'):
1039
factory = KnitPlainFactory()
1043
index = _KnitIndex(get_transport('.'), name + INDEX_SUFFIX,
1044
access_mode, create=True, file_mode=None,
1045
create_parent_dir=False, delay_create=delay_create,
1046
dir_mode=None, get_scope=lambda:None)
1047
access = _KnitAccess(get_transport('.'), name + DATA_SUFFIX, None,
1048
None, delay_create, False)
1049
return KnitVersionedFile(name, get_transport('.'), factory=factory,
1050
create=True, delay_create=delay_create, index=index,
1051
access_method=access, delta=delta)
1053
def assertRecordContentEqual(self, knit, version_id, candidate_content):
1054
"""Assert that some raw record content matches the raw record content
1055
for a particular version_id in the given knit.
1057
index_memo = knit._index.get_position(version_id)
1058
record = (version_id, index_memo)
1059
[(_, expected_content)] = list(knit._data.read_records_iter_raw([record]))
1060
self.assertEqual(expected_content, candidate_content)
1063
class BasicKnitTests(KnitTests):
1065
def add_stock_one_and_one_a(self, k):
1066
k.add_lines('text-1', [], split_lines(TEXT_1))
1067
k.add_lines('text-1a', ['text-1'], split_lines(TEXT_1A))
1069
def test_knit_constructor(self):
1070
"""Construct empty k"""
1071
self.make_test_knit()
1073
def test_make_explicit_index(self):
1074
"""We can supply an index to use."""
1075
knit = KnitVersionedFile('test', get_transport('.'),
1076
index='strangelove', access_method="a")
1077
self.assertEqual(knit._index, 'strangelove')
1079
def test_knit_add(self):
1080
"""Store one text in knit and retrieve"""
1081
k = self.make_test_knit()
1082
k.add_lines('text-1', [], split_lines(TEXT_1))
1083
self.assertTrue(k.has_version('text-1'))
1084
self.assertEqualDiff(''.join(k.get_lines('text-1')), TEXT_1)
1086
def test_newline_empty_lines(self):
1087
# ensure that ["\n"] round trips ok.
1088
knit = self.make_test_knit()
1089
knit.add_lines('a', [], ["\n"])
1090
knit.add_lines_with_ghosts('b', [], ["\n"])
1091
self.assertEqual(["\n"], knit.get_lines('a'))
1092
self.assertEqual(["\n"], knit.get_lines('b'))
1093
self.assertEqual(['fulltext'], knit._index.get_options('a'))
1094
self.assertEqual(['fulltext'], knit._index.get_options('b'))
1095
knit.add_lines('c', ['a'], ["\n"])
1096
knit.add_lines_with_ghosts('d', ['b'], ["\n"])
1097
self.assertEqual(["\n"], knit.get_lines('c'))
1098
self.assertEqual(["\n"], knit.get_lines('d'))
1099
self.assertEqual(['line-delta'], knit._index.get_options('c'))
1100
self.assertEqual(['line-delta'], knit._index.get_options('d'))
1102
def test_empty_lines(self):
1103
# bizarrely, [] is not listed as having no-eol.
1104
knit = self.make_test_knit()
1105
knit.add_lines('a', [], [])
1106
knit.add_lines_with_ghosts('b', [], [])
1107
self.assertEqual([], knit.get_lines('a'))
1108
self.assertEqual([], knit.get_lines('b'))
1109
self.assertEqual(['fulltext'], knit._index.get_options('a'))
1110
self.assertEqual(['fulltext'], knit._index.get_options('b'))
1111
knit.add_lines('c', ['a'], [])
1112
knit.add_lines_with_ghosts('d', ['b'], [])
1113
self.assertEqual([], knit.get_lines('c'))
1114
self.assertEqual([], knit.get_lines('d'))
1115
self.assertEqual(['line-delta'], knit._index.get_options('c'))
1116
self.assertEqual(['line-delta'], knit._index.get_options('d'))
1118
def test_knit_reload(self):
1119
# test that the content in a reloaded knit is correct
1120
k = self.make_test_knit()
1121
k.add_lines('text-1', [], split_lines(TEXT_1))
1123
k2 = make_file_knit('test', get_transport('.'), access_mode='r',
1124
factory=KnitPlainFactory(), create=True)
1125
self.assertTrue(k2.has_version('text-1'))
1126
self.assertEqualDiff(''.join(k2.get_lines('text-1')), TEXT_1)
1128
def test_knit_several(self):
1129
"""Store several texts in a knit"""
1130
k = self.make_test_knit()
1131
k.add_lines('text-1', [], split_lines(TEXT_1))
1132
k.add_lines('text-2', [], split_lines(TEXT_2))
1133
self.assertEqualDiff(''.join(k.get_lines('text-1')), TEXT_1)
1134
self.assertEqualDiff(''.join(k.get_lines('text-2')), TEXT_2)
1136
def test_repeated_add(self):
1137
"""Knit traps attempt to replace existing version"""
1138
k = self.make_test_knit()
1139
k.add_lines('text-1', [], split_lines(TEXT_1))
1140
self.assertRaises(RevisionAlreadyPresent,
1142
'text-1', [], split_lines(TEXT_1))
1144
def test_empty(self):
1145
k = self.make_test_knit(True)
1146
k.add_lines('text-1', [], [])
1147
self.assertEquals(k.get_lines('text-1'), [])
1149
def test_incomplete(self):
1150
"""Test if texts without a ending line-end can be inserted and
1152
k = make_file_knit('test', get_transport('.'), delta=False, create=True)
1153
k.add_lines('text-1', [], ['a\n', 'b' ])
1154
k.add_lines('text-2', ['text-1'], ['a\rb\n', 'b\n'])
1155
# reopening ensures maximum room for confusion
1156
k = make_file_knit('test', get_transport('.'), delta=False, create=True)
1157
self.assertEquals(k.get_lines('text-1'), ['a\n', 'b' ])
1158
self.assertEquals(k.get_lines('text-2'), ['a\rb\n', 'b\n'])
1160
def test_delta(self):
1161
"""Expression of knit delta as lines"""
1162
k = self.make_test_knit()
1163
td = list(line_delta(TEXT_1.splitlines(True),
1164
TEXT_1A.splitlines(True)))
1165
self.assertEqualDiff(''.join(td), delta_1_1a)
1166
out = apply_line_delta(TEXT_1.splitlines(True), td)
1167
self.assertEqualDiff(''.join(out), TEXT_1A)
1169
def test_add_with_parents(self):
1170
"""Store in knit with parents"""
1171
k = self.make_test_knit()
1172
self.add_stock_one_and_one_a(k)
1173
self.assertEqual({'text-1':(), 'text-1a':('text-1',)},
1174
k.get_parent_map(['text-1', 'text-1a']))
1176
def test_ancestry(self):
1177
"""Store in knit with parents"""
1178
k = self.make_test_knit()
1179
self.add_stock_one_and_one_a(k)
1180
self.assertEquals(set(k.get_ancestry(['text-1a'])), set(['text-1a', 'text-1']))
1182
def test_add_delta(self):
1183
"""Store in knit with parents"""
1184
k = self.make_test_knit(annotate=False)
1185
self.add_stock_one_and_one_a(k)
1186
self.assertEqualDiff(''.join(k.get_lines('text-1a')), TEXT_1A)
1188
def test_add_delta_knit_graph_index(self):
1189
"""Does adding work with a KnitGraphIndex."""
1190
index = InMemoryGraphIndex(2)
1191
knit_index = KnitGraphIndex(index, add_callback=index.add_nodes,
1193
k = self.make_test_knit(annotate=True, index=knit_index)
1194
self.add_stock_one_and_one_a(k)
1195
self.assertEqualDiff(''.join(k.get_lines('text-1a')), TEXT_1A)
1196
# check the index had the right data added.
1197
self.assertEqual(set([
1198
(index, ('text-1', ), ' 0 127', ((), ())),
1199
(index, ('text-1a', ), ' 127 140', ((('text-1', ),), (('text-1', ),))),
1200
]), set(index.iter_all_entries()))
1201
# we should not have a .kndx file
1202
self.assertFalse(get_transport('.').has('test.kndx'))
1204
def test_annotate(self):
1206
k = self.make_test_knit(annotate=True, name='knit')
1207
self.insert_and_test_small_annotate(k)
1209
def insert_and_test_small_annotate(self, k):
1210
"""test annotation with k works correctly."""
1211
k.add_lines('text-1', [], ['a\n', 'b\n'])
1212
k.add_lines('text-2', ['text-1'], ['a\n', 'c\n'])
1214
origins = k.annotate('text-2')
1215
self.assertEquals(origins[0], ('text-1', 'a\n'))
1216
self.assertEquals(origins[1], ('text-2', 'c\n'))
1218
def test_annotate_fulltext(self):
1220
k = self.make_test_knit(annotate=True, name='knit', delta=False)
1221
self.insert_and_test_small_annotate(k)
1223
def test_annotate_merge_1(self):
1224
k = self.make_test_knit(True)
1225
k.add_lines('text-a1', [], ['a\n', 'b\n'])
1226
k.add_lines('text-a2', [], ['d\n', 'c\n'])
1227
k.add_lines('text-am', ['text-a1', 'text-a2'], ['d\n', 'b\n'])
1228
origins = k.annotate('text-am')
1229
self.assertEquals(origins[0], ('text-a2', 'd\n'))
1230
self.assertEquals(origins[1], ('text-a1', 'b\n'))
1232
def test_annotate_merge_2(self):
1233
k = self.make_test_knit(True)
1234
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1235
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1236
k.add_lines('text-am', ['text-a1', 'text-a2'], ['a\n', 'y\n', 'c\n'])
1237
origins = k.annotate('text-am')
1238
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1239
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1240
self.assertEquals(origins[2], ('text-a1', 'c\n'))
1242
def test_annotate_merge_9(self):
1243
k = self.make_test_knit(True)
1244
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1245
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1246
k.add_lines('text-am', ['text-a1', 'text-a2'], ['k\n', 'y\n', 'c\n'])
1247
origins = k.annotate('text-am')
1248
self.assertEquals(origins[0], ('text-am', 'k\n'))
1249
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1250
self.assertEquals(origins[2], ('text-a1', 'c\n'))
1252
def test_annotate_merge_3(self):
1253
k = self.make_test_knit(True)
1254
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1255
k.add_lines('text-a2', [] ,['x\n', 'y\n', 'z\n'])
1256
k.add_lines('text-am', ['text-a1', 'text-a2'], ['k\n', 'y\n', 'z\n'])
1257
origins = k.annotate('text-am')
1258
self.assertEquals(origins[0], ('text-am', 'k\n'))
1259
self.assertEquals(origins[1], ('text-a2', 'y\n'))
1260
self.assertEquals(origins[2], ('text-a2', 'z\n'))
1262
def test_annotate_merge_4(self):
1263
k = self.make_test_knit(True)
1264
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1265
k.add_lines('text-a2', [], ['x\n', 'y\n', 'z\n'])
1266
k.add_lines('text-a3', ['text-a1'], ['a\n', 'b\n', 'p\n'])
1267
k.add_lines('text-am', ['text-a2', 'text-a3'], ['a\n', 'b\n', 'z\n'])
1268
origins = k.annotate('text-am')
1269
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1270
self.assertEquals(origins[1], ('text-a1', 'b\n'))
1271
self.assertEquals(origins[2], ('text-a2', 'z\n'))
1273
def test_annotate_merge_5(self):
1274
k = self.make_test_knit(True)
1275
k.add_lines('text-a1', [], ['a\n', 'b\n', 'c\n'])
1276
k.add_lines('text-a2', [], ['d\n', 'e\n', 'f\n'])
1277
k.add_lines('text-a3', [], ['x\n', 'y\n', 'z\n'])
1278
k.add_lines('text-am',
1279
['text-a1', 'text-a2', 'text-a3'],
1280
['a\n', 'e\n', 'z\n'])
1281
origins = k.annotate('text-am')
1282
self.assertEquals(origins[0], ('text-a1', 'a\n'))
1283
self.assertEquals(origins[1], ('text-a2', 'e\n'))
1284
self.assertEquals(origins[2], ('text-a3', 'z\n'))
1286
def test_annotate_file_cherry_pick(self):
1287
k = self.make_test_knit(True)
1288
k.add_lines('text-1', [], ['a\n', 'b\n', 'c\n'])
1289
k.add_lines('text-2', ['text-1'], ['d\n', 'e\n', 'f\n'])
1290
k.add_lines('text-3', ['text-2', 'text-1'], ['a\n', 'b\n', 'c\n'])
1291
origins = k.annotate('text-3')
1292
self.assertEquals(origins[0], ('text-1', 'a\n'))
1293
self.assertEquals(origins[1], ('text-1', 'b\n'))
1294
self.assertEquals(origins[2], ('text-1', 'c\n'))
1296
def _test_join_with_factories(self, k1_factory, k2_factory):
1297
k1 = make_file_knit('test1', get_transport('.'), factory=k1_factory, create=True)
1298
k1.add_lines('text-a', [], ['a1\n', 'a2\n', 'a3\n'])
1299
k1.add_lines('text-b', ['text-a'], ['a1\n', 'b2\n', 'a3\n'])
1300
k1.add_lines('text-c', [], ['c1\n', 'c2\n', 'c3\n'])
1301
k1.add_lines('text-d', ['text-c'], ['c1\n', 'd2\n', 'd3\n'])
1302
k1.add_lines('text-m', ['text-b', 'text-d'], ['a1\n', 'b2\n', 'd3\n'])
1303
k2 = make_file_knit('test2', get_transport('.'), factory=k2_factory, create=True)
1304
count = k2.join(k1, version_ids=['text-m'])
1305
self.assertEquals(count, 5)
1306
self.assertTrue(k2.has_version('text-a'))
1307
self.assertTrue(k2.has_version('text-c'))
1308
origins = k2.annotate('text-m')
1309
self.assertEquals(origins[0], ('text-a', 'a1\n'))
1310
self.assertEquals(origins[1], ('text-b', 'b2\n'))
1311
self.assertEquals(origins[2], ('text-d', 'd3\n'))
1313
def test_knit_join_plain_to_plain(self):
1314
"""Test joining a plain knit with a plain knit."""
1315
self._test_join_with_factories(KnitPlainFactory(), KnitPlainFactory())
1317
def test_knit_join_anno_to_anno(self):
1318
"""Test joining an annotated knit with an annotated knit."""
1319
self._test_join_with_factories(None, None)
1321
def test_knit_join_anno_to_plain(self):
1322
"""Test joining an annotated knit with a plain knit."""
1323
self._test_join_with_factories(None, KnitPlainFactory())
1325
def test_knit_join_plain_to_anno(self):
1326
"""Test joining a plain knit with an annotated knit."""
1327
self._test_join_with_factories(KnitPlainFactory(), None)
1329
def test_reannotate(self):
1330
k1 = make_file_knit('knit1', get_transport('.'),
1331
factory=KnitAnnotateFactory(), create=True)
1333
k1.add_lines('text-a', [], ['a\n', 'b\n'])
1335
k1.add_lines('text-b', ['text-a'], ['a\n', 'c\n'])
1337
k2 = make_file_knit('test2', get_transport('.'),
1338
factory=KnitAnnotateFactory(), create=True)
1339
k2.join(k1, version_ids=['text-b'])
1342
k1.add_lines('text-X', ['text-b'], ['a\n', 'b\n'])
1344
k2.add_lines('text-c', ['text-b'], ['z\n', 'c\n'])
1346
k2.add_lines('text-Y', ['text-b'], ['b\n', 'c\n'])
1348
# test-c will have index 3
1349
k1.join(k2, version_ids=['text-c'])
1351
lines = k1.get_lines('text-c')
1352
self.assertEquals(lines, ['z\n', 'c\n'])
1354
origins = k1.annotate('text-c')
1355
self.assertEquals(origins[0], ('text-c', 'z\n'))
1356
self.assertEquals(origins[1], ('text-b', 'c\n'))
1358
def test_get_line_delta_texts(self):
1359
"""Make sure we can call get_texts on text with reused line deltas"""
1360
k1 = make_file_knit('test1', get_transport('.'),
1361
factory=KnitPlainFactory(), create=True)
1366
parents = ['%d' % (t-1)]
1367
k1.add_lines('%d' % t, parents, ['hello\n'] * t)
1368
k1.get_texts(('%d' % t) for t in range(3))
1370
def test_iter_lines_reads_in_order(self):
1371
instrumented_t = get_transport('trace+memory:///')
1372
k1 = make_file_knit('id', instrumented_t, create=True, delta=True)
1373
self.assertEqual([('get', 'id.kndx',)], instrumented_t._activity)
1374
# add texts with no required ordering
1375
k1.add_lines('base', [], ['text\n'])
1376
k1.add_lines('base2', [], ['text2\n'])
1377
# clear the logged activity, but preserve the list instance in case of
1378
# clones pointing at it.
1379
del instrumented_t._activity[:]
1380
# request a last-first iteration
1381
results = list(k1.iter_lines_added_or_present_in_versions(
1384
[('readv', 'id.knit', [(0, 87), (87, 89)], False, None)],
1385
instrumented_t._activity)
1386
self.assertEqual([('text\n', 'base'), ('text2\n', 'base2')], results)
1388
def test_knit_format(self):
1389
# this tests that a new knit index file has the expected content
1390
# and that is writes the data we expect as records are added.
1391
knit = self.make_test_knit(True)
1392
# Now knit files are not created until we first add data to them
1393
self.assertFileEqual("# bzr knit index 8\n", 'test.kndx')
1394
knit.add_lines_with_ghosts('revid', ['a_ghost'], ['a\n'])
1395
self.assertFileEqual(
1396
"# bzr knit index 8\n"
1398
"revid fulltext 0 84 .a_ghost :",
1400
knit.add_lines_with_ghosts('revid2', ['revid'], ['a\n'])
1401
self.assertFileEqual(
1402
"# bzr knit index 8\n"
1403
"\nrevid fulltext 0 84 .a_ghost :"
1404
"\nrevid2 line-delta 84 82 0 :",
1406
# we should be able to load this file again
1407
knit = make_file_knit('test', get_transport('.'), access_mode='r')
1408
self.assertEqual(['revid', 'revid2'], knit.versions())
1409
# write a short write to the file and ensure that its ignored
1410
indexfile = file('test.kndx', 'ab')
1411
indexfile.write('\nrevid3 line-delta 166 82 1 2 3 4 5 .phwoar:demo ')
1413
# we should be able to load this file again
1414
knit = make_file_knit('test', get_transport('.'), access_mode='w')
1415
self.assertEqual(['revid', 'revid2'], knit.versions())
1416
# and add a revision with the same id the failed write had
1417
knit.add_lines('revid3', ['revid2'], ['a\n'])
1418
# and when reading it revid3 should now appear.
1419
knit = make_file_knit('test', get_transport('.'), access_mode='r')
1420
self.assertEqual(['revid', 'revid2', 'revid3'], knit.versions())
1421
self.assertEqual({'revid3':('revid2',)}, knit.get_parent_map(['revid3']))
1423
def test_delay_create(self):
1424
"""Test that passing delay_create=True creates files late"""
1425
knit = self.make_test_knit(annotate=True, delay_create=True)
1426
self.failIfExists('test.knit')
1427
self.failIfExists('test.kndx')
1428
knit.add_lines_with_ghosts('revid', ['a_ghost'], ['a\n'])
1429
self.failUnlessExists('test.knit')
1430
self.assertFileEqual(
1431
"# bzr knit index 8\n"
1433
"revid fulltext 0 84 .a_ghost :",
1436
def test_create_parent_dir(self):
1437
"""create_parent_dir can create knits in nonexistant dirs"""
1438
# Has no effect if we don't set 'delay_create'
1439
trans = get_transport('.')
1440
self.assertRaises(NoSuchFile, make_file_knit, 'dir/test',
1441
trans, access_mode='w', factory=None,
1442
create=True, create_parent_dir=True)
1443
# Nothing should have changed yet
1444
knit = make_file_knit('dir/test', trans, access_mode='w',
1445
factory=None, create=True,
1446
create_parent_dir=True,
1448
self.failIfExists('dir/test.knit')
1449
self.failIfExists('dir/test.kndx')
1450
self.failIfExists('dir')
1451
knit.add_lines('revid', [], ['a\n'])
1452
self.failUnlessExists('dir')
1453
self.failUnlessExists('dir/test.knit')
1454
self.assertFileEqual(
1455
"# bzr knit index 8\n"
1457
"revid fulltext 0 84 :",
1460
def test_create_mode_700(self):
1461
trans = get_transport('.')
1462
if not trans._can_roundtrip_unix_modebits():
1463
# Can't roundtrip, so no need to run this test
1465
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1466
create=True, create_parent_dir=True, delay_create=True,
1467
file_mode=0600, dir_mode=0700)
1468
knit.add_lines('revid', [], ['a\n'])
1469
self.assertTransportMode(trans, 'dir', 0700)
1470
self.assertTransportMode(trans, 'dir/test.knit', 0600)
1471
self.assertTransportMode(trans, 'dir/test.kndx', 0600)
1473
def test_create_mode_770(self):
1474
trans = get_transport('.')
1475
if not trans._can_roundtrip_unix_modebits():
1476
# Can't roundtrip, so no need to run this test
1478
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1479
create=True, create_parent_dir=True, delay_create=True,
1480
file_mode=0660, dir_mode=0770)
1481
knit.add_lines('revid', [], ['a\n'])
1482
self.assertTransportMode(trans, 'dir', 0770)
1483
self.assertTransportMode(trans, 'dir/test.knit', 0660)
1484
self.assertTransportMode(trans, 'dir/test.kndx', 0660)
1486
def test_create_mode_777(self):
1487
trans = get_transport('.')
1488
if not trans._can_roundtrip_unix_modebits():
1489
# Can't roundtrip, so no need to run this test
1491
knit = make_file_knit('dir/test', trans, access_mode='w', factory=None,
1492
create=True, create_parent_dir=True, delay_create=True,
1493
file_mode=0666, dir_mode=0777)
1494
knit.add_lines('revid', [], ['a\n'])
1495
self.assertTransportMode(trans, 'dir', 0777)
1496
self.assertTransportMode(trans, 'dir/test.knit', 0666)
1497
self.assertTransportMode(trans, 'dir/test.kndx', 0666)
1499
def test_plan_merge(self):
1500
my_knit = self.make_test_knit(annotate=True)
1501
my_knit.add_lines('text1', [], split_lines(TEXT_1))
1502
my_knit.add_lines('text1a', ['text1'], split_lines(TEXT_1A))
1503
my_knit.add_lines('text1b', ['text1'], split_lines(TEXT_1B))
1504
plan = list(my_knit.plan_merge('text1a', 'text1b'))
1505
for plan_line, expected_line in zip(plan, AB_MERGE):
1506
self.assertEqual(plan_line, expected_line)
1509
class GetDataStreamTests(KnitTests):
1510
"""Tests for get_data_stream."""
1512
def test_get_stream_empty(self):
1513
"""Get a data stream for an empty knit file."""
1514
k1 = self.make_test_knit()
1515
format, data_list, reader_callable = k1.get_data_stream([])
1516
self.assertEqual('knit-plain', format)
1517
self.assertEqual([], data_list)
1518
content = reader_callable(None)
1519
self.assertEqual('', content)
1520
self.assertIsInstance(content, str)
1522
def test_get_stream_one_version(self):
1523
"""Get a data stream for a single record out of a knit containing just
1526
k1 = self.make_test_knit()
1528
('text-a', [], TEXT_1),
1530
expected_data_list = [
1531
# version, options, length, parents
1532
('text-a', ['fulltext'], 122, ()),
1534
for version_id, parents, lines in test_data:
1535
k1.add_lines(version_id, parents, split_lines(lines))
1537
format, data_list, reader_callable = k1.get_data_stream(['text-a'])
1538
self.assertEqual('knit-plain', format)
1539
self.assertEqual(expected_data_list, data_list)
1540
# There's only one record in the knit, so the content should be the
1541
# entire knit data file's contents.
1542
self.assertEqual(k1.transport.get_bytes(k1._data._access._filename),
1543
reader_callable(None))
1545
def test_get_stream_get_one_version_of_many(self):
1546
"""Get a data stream for just one version out of a knit containing many
1549
k1 = self.make_test_knit()
1550
# Insert the same data as test_knit_join, as they seem to cover a range
1551
# of cases (no parents, one parent, multiple parents).
1553
('text-a', [], TEXT_1),
1554
('text-b', ['text-a'], TEXT_1),
1555
('text-c', [], TEXT_1),
1556
('text-d', ['text-c'], TEXT_1),
1557
('text-m', ['text-b', 'text-d'], TEXT_1),
1559
expected_data_list = [
1560
# version, options, length, parents
1561
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1563
for version_id, parents, lines in test_data:
1564
k1.add_lines(version_id, parents, split_lines(lines))
1566
format, data_list, reader_callable = k1.get_data_stream(['text-m'])
1567
self.assertEqual('knit-plain', format)
1568
self.assertEqual(expected_data_list, data_list)
1569
self.assertRecordContentEqual(k1, 'text-m', reader_callable(None))
1571
def test_get_data_stream_unordered_index(self):
1572
"""Get a data stream when the knit index reports versions out of order.
1574
https://bugs.launchpad.net/bzr/+bug/164637
1576
k1 = self.make_test_knit()
1578
('text-a', [], TEXT_1),
1579
('text-b', ['text-a'], TEXT_1),
1580
('text-c', [], TEXT_1),
1581
('text-d', ['text-c'], TEXT_1),
1582
('text-m', ['text-b', 'text-d'], TEXT_1),
1584
for version_id, parents, lines in test_data:
1585
k1.add_lines(version_id, parents, split_lines(lines))
1586
# monkey-patch versions method to return out of order, as if coming
1587
# from multiple independently indexed packs
1588
original_versions = k1.versions
1589
k1.versions = lambda: reversed(original_versions())
1590
expected_data_list = [
1591
('text-a', ['fulltext'], 122, ()),
1592
('text-b', ['line-delta'], 84, ('text-a',))]
1593
# now check the fulltext is first and the delta second
1594
format, data_list, _ = k1.get_data_stream(['text-a', 'text-b'])
1595
self.assertEqual('knit-plain', format)
1596
self.assertEqual(expected_data_list, data_list)
1597
# and that's true if we ask for them in the opposite order too
1598
format, data_list, _ = k1.get_data_stream(['text-b', 'text-a'])
1599
self.assertEqual(expected_data_list, data_list)
1600
# also try requesting more versions
1601
format, data_list, _ = k1.get_data_stream([
1602
'text-m', 'text-b', 'text-a'])
1604
('text-a', ['fulltext'], 122, ()),
1605
('text-b', ['line-delta'], 84, ('text-a',)),
1606
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1609
def test_get_stream_ghost_parent(self):
1610
"""Get a data stream for a version with a ghost parent."""
1611
k1 = self.make_test_knit()
1613
k1.add_lines('text-a', [], split_lines(TEXT_1))
1614
k1.add_lines_with_ghosts('text-b', ['text-a', 'text-ghost'],
1615
split_lines(TEXT_1))
1617
expected_data_list = [
1618
# version, options, length, parents
1619
('text-b', ['line-delta'], 84, ('text-a', 'text-ghost')),
1622
format, data_list, reader_callable = k1.get_data_stream(['text-b'])
1623
self.assertEqual('knit-plain', format)
1624
self.assertEqual(expected_data_list, data_list)
1625
self.assertRecordContentEqual(k1, 'text-b', reader_callable(None))
1627
def test_get_stream_get_multiple_records(self):
1628
"""Get a stream for multiple records of a knit."""
1629
k1 = self.make_test_knit()
1630
# Insert the same data as test_knit_join, as they seem to cover a range
1631
# of cases (no parents, one parent, multiple parents).
1633
('text-a', [], TEXT_1),
1634
('text-b', ['text-a'], TEXT_1),
1635
('text-c', [], TEXT_1),
1636
('text-d', ['text-c'], TEXT_1),
1637
('text-m', ['text-b', 'text-d'], TEXT_1),
1639
for version_id, parents, lines in test_data:
1640
k1.add_lines(version_id, parents, split_lines(lines))
1642
# This test is actually a bit strict as the order in which they're
1643
# returned is not defined. This matches the current (deterministic)
1645
expected_data_list = [
1646
# version, options, length, parents
1647
('text-d', ['line-delta'], 84, ('text-c',)),
1648
('text-b', ['line-delta'], 84, ('text-a',)),
1650
# Note that even though we request the revision IDs in a particular
1651
# order, the data stream may return them in any order it likes. In this
1652
# case, they'll be in the order they were inserted into the knit.
1653
format, data_list, reader_callable = k1.get_data_stream(
1654
['text-d', 'text-b'])
1655
self.assertEqual('knit-plain', format)
1656
self.assertEqual(expected_data_list, data_list)
1657
# must match order they're returned
1658
self.assertRecordContentEqual(k1, 'text-d', reader_callable(84))
1659
self.assertRecordContentEqual(k1, 'text-b', reader_callable(84))
1660
self.assertEqual('', reader_callable(None),
1661
"There should be no more bytes left to read.")
1663
def test_get_stream_all(self):
1664
"""Get a data stream for all the records in a knit.
1666
This exercises fulltext records, line-delta records, records with
1667
various numbers of parents, and reading multiple records out of the
1668
callable. These cases ought to all be exercised individually by the
1669
other test_get_stream_* tests; this test is basically just paranoia.
1671
k1 = self.make_test_knit()
1672
# Insert the same data as test_knit_join, as they seem to cover a range
1673
# of cases (no parents, one parent, multiple parents).
1675
('text-a', [], TEXT_1),
1676
('text-b', ['text-a'], TEXT_1),
1677
('text-c', [], TEXT_1),
1678
('text-d', ['text-c'], TEXT_1),
1679
('text-m', ['text-b', 'text-d'], TEXT_1),
1681
for version_id, parents, lines in test_data:
1682
k1.add_lines(version_id, parents, split_lines(lines))
1684
# This test is actually a bit strict as the order in which they're
1685
# returned is not defined. This matches the current (deterministic)
1687
expected_data_list = [
1688
# version, options, length, parents
1689
('text-a', ['fulltext'], 122, ()),
1690
('text-b', ['line-delta'], 84, ('text-a',)),
1691
('text-m', ['line-delta'], 84, ('text-b', 'text-d')),
1692
('text-c', ['fulltext'], 121, ()),
1693
('text-d', ['line-delta'], 84, ('text-c',)),
1695
format, data_list, reader_callable = k1.get_data_stream(
1696
['text-a', 'text-b', 'text-c', 'text-d', 'text-m'])
1697
self.assertEqual('knit-plain', format)
1698
self.assertEqual(expected_data_list, data_list)
1699
for version_id, options, length, parents in expected_data_list:
1700
bytes = reader_callable(length)
1701
self.assertRecordContentEqual(k1, version_id, bytes)
1704
class InsertDataStreamTests(KnitTests):
1705
"""Tests for insert_data_stream."""
1707
def assertKnitFilesEqual(self, knit1, knit2):
1708
"""Assert that the contents of the index and data files of two knits are
1712
knit1.transport.get_bytes(knit1._data._access._filename),
1713
knit2.transport.get_bytes(knit2._data._access._filename))
1715
knit1.transport.get_bytes(knit1._index._filename),
1716
knit2.transport.get_bytes(knit2._index._filename))
1718
def assertKnitValuesEqual(self, left, right):
1719
"""Assert that the texts, annotations and graph of left and right are
1722
self.assertEqual(set(left.versions()), set(right.versions()))
1723
for version in left.versions():
1724
self.assertEqual(left.get_parents_with_ghosts(version),
1725
right.get_parents_with_ghosts(version))
1726
self.assertEqual(left.get_lines(version),
1727
right.get_lines(version))
1728
self.assertEqual(left.annotate(version),
1729
right.annotate(version))
1731
def test_empty_stream(self):
1732
"""Inserting a data stream with no records should not put any data into
1735
k1 = self.make_test_knit()
1736
k1.insert_data_stream(
1737
(k1.get_format_signature(), [], lambda ignored: ''))
1738
self.assertEqual('', k1.transport.get_bytes(k1._data._access._filename),
1739
"The .knit should be completely empty.")
1740
self.assertEqual(k1._index.HEADER,
1741
k1.transport.get_bytes(k1._index._filename),
1742
"The .kndx should have nothing apart from the header.")
1744
def test_one_record(self):
1745
"""Inserting a data stream with one record from a knit with one record
1746
results in byte-identical files.
1748
source = self.make_test_knit(name='source')
1749
source.add_lines('text-a', [], split_lines(TEXT_1))
1750
data_stream = source.get_data_stream(['text-a'])
1751
target = self.make_test_knit(name='target')
1752
target.insert_data_stream(data_stream)
1753
self.assertKnitFilesEqual(source, target)
1755
def test_annotated_stream_into_unannotated_knit(self):
1756
"""Inserting an annotated datastream to an unannotated knit works."""
1757
# case one - full texts.
1758
source = self.make_test_knit(name='source', annotate=True)
1759
target = self.make_test_knit(name='target', annotate=False)
1760
source.add_lines('text-a', [], split_lines(TEXT_1))
1761
target.insert_data_stream(source.get_data_stream(['text-a']))
1762
self.assertKnitValuesEqual(source, target)
1763
# case two - deltas.
1764
source.add_lines('text-b', ['text-a'], split_lines(TEXT_2))
1765
target.insert_data_stream(source.get_data_stream(['text-b']))
1766
self.assertKnitValuesEqual(source, target)
1768
def test_unannotated_stream_into_annotated_knit(self):
1769
"""Inserting an unannotated datastream to an annotated knit works."""
1770
# case one - full texts.
1771
source = self.make_test_knit(name='source', annotate=False)
1772
target = self.make_test_knit(name='target', annotate=True)
1773
source.add_lines('text-a', [], split_lines(TEXT_1))
1774
target.insert_data_stream(source.get_data_stream(['text-a']))
1775
self.assertKnitValuesEqual(source, target)
1776
# case two - deltas.
1777
source.add_lines('text-b', ['text-a'], split_lines(TEXT_2))
1778
target.insert_data_stream(source.get_data_stream(['text-b']))
1779
self.assertKnitValuesEqual(source, target)
1781
def test_records_already_present(self):
1782
"""Insert a data stream where some records are alreday present in the
1783
target, and some not. Only the new records are inserted.
1785
source = self.make_test_knit(name='source')
1786
target = self.make_test_knit(name='target')
1787
# Insert 'text-a' into both source and target
1788
source.add_lines('text-a', [], split_lines(TEXT_1))
1789
target.insert_data_stream(source.get_data_stream(['text-a']))
1790
# Insert 'text-b' into just the source.
1791
source.add_lines('text-b', ['text-a'], split_lines(TEXT_1))
1792
# Get a data stream of both text-a and text-b, and insert it.
1793
data_stream = source.get_data_stream(['text-a', 'text-b'])
1794
target.insert_data_stream(data_stream)
1795
# The source and target will now be identical. This means the text-a
1796
# record was not added a second time.
1797
self.assertKnitFilesEqual(source, target)
1799
def test_multiple_records(self):
1800
"""Inserting a data stream of all records from a knit with multiple
1801
records results in byte-identical files.
1803
source = self.make_test_knit(name='source')
1804
source.add_lines('text-a', [], split_lines(TEXT_1))
1805
source.add_lines('text-b', ['text-a'], split_lines(TEXT_1))
1806
source.add_lines('text-c', [], split_lines(TEXT_1))
1807
data_stream = source.get_data_stream(['text-a', 'text-b', 'text-c'])
1809
target = self.make_test_knit(name='target')
1810
target.insert_data_stream(data_stream)
1812
self.assertKnitFilesEqual(source, target)
1814
def test_ghost_parent(self):
1815
"""Insert a data stream with a record that has a ghost parent."""
1816
# Make a knit with a record, text-a, that has a ghost parent.
1817
source = self.make_test_knit(name='source')
1818
source.add_lines_with_ghosts('text-a', ['text-ghost'],
1819
split_lines(TEXT_1))
1820
data_stream = source.get_data_stream(['text-a'])
1822
target = self.make_test_knit(name='target')
1823
target.insert_data_stream(data_stream)
1825
self.assertKnitFilesEqual(source, target)
1827
# The target knit object is in a consistent state, i.e. the record we
1828
# just added is immediately visible.
1829
self.assertTrue(target.has_version('text-a'))
1830
self.assertFalse(target.has_version('text-ghost'))
1831
self.assertEqual({'text-a':('text-ghost',)},
1832
target.get_parent_map(['text-a', 'text-ghost']))
1833
self.assertEqual(split_lines(TEXT_1), target.get_lines('text-a'))
1835
def test_inconsistent_version_lines(self):
1836
"""Inserting a data stream which has different content for a version_id
1837
than already exists in the knit will raise KnitCorrupt.
1839
source = self.make_test_knit(name='source')
1840
target = self.make_test_knit(name='target')
1841
# Insert a different 'text-a' into both source and target
1842
source.add_lines('text-a', [], split_lines(TEXT_1))
1843
target.add_lines('text-a', [], split_lines(TEXT_2))
1844
# Insert a data stream with conflicting content into the target
1845
data_stream = source.get_data_stream(['text-a'])
1847
errors.KnitCorrupt, target.insert_data_stream, data_stream)
1849
def test_inconsistent_version_parents(self):
1850
"""Inserting a data stream which has different parents for a version_id
1851
than already exists in the knit will raise KnitCorrupt.
1853
source = self.make_test_knit(name='source')
1854
target = self.make_test_knit(name='target')
1855
# Insert a different 'text-a' into both source and target. They differ
1856
# only by the parents list, the content is the same.
1857
source.add_lines_with_ghosts('text-a', [], split_lines(TEXT_1))
1858
target.add_lines_with_ghosts('text-a', ['a-ghost'], split_lines(TEXT_1))
1859
# Insert a data stream with conflicting content into the target
1860
data_stream = source.get_data_stream(['text-a'])
1862
errors.KnitCorrupt, target.insert_data_stream, data_stream)
1864
def test_unknown_stream_format(self):
1865
"""A data stream in a different format to the target knit cannot be
1868
It will raise KnitDataStreamUnknown because the fallback code will fail
1869
to make a knit. In future we may need KnitDataStreamIncompatible again,
1870
for more exotic cases.
1872
data_stream = ('fake-format-signature', [], lambda _: '')
1873
target = self.make_test_knit(name='target')
1875
errors.KnitDataStreamUnknown,
1876
target.insert_data_stream, data_stream)
1878
def test_bug_208418(self):
1879
"""You can insert a stream with an incompatible format, even when:
1880
* the stream has a line-delta record,
1881
* whose parent is in the target, also stored as a line-delta
1883
See <https://launchpad.net/bugs/208418>.
1885
base_lines = split_lines(TEXT_1)
1887
target = self.make_test_knit(name='target', annotate=True)
1888
target.add_lines('version-1', [], base_lines)
1889
target.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1890
# The second record should be a delta.
1891
self.assertEqual('line-delta', target._index.get_method('version-2'))
1893
# Make a source, with a different format, but the same data
1894
source = self.make_test_knit(name='source', annotate=False)
1895
source.add_lines('version-1', [], base_lines)
1896
source.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1897
# Now add another record, which should be stored as a delta against
1899
source.add_lines('version-3', ['version-2'], base_lines + ['b\n'])
1900
self.assertEqual('line-delta', source._index.get_method('version-3'))
1902
# Make a stream of the new version
1903
data_stream = source.get_data_stream(['version-3'])
1904
# And insert into the target
1905
target.insert_data_stream(data_stream)
1906
# No errors should have been raised.
1908
def test_line_delta_record_into_non_delta_knit(self):
1909
# Make a data stream with a line-delta record
1910
source = self.make_test_knit(name='source', delta=True)
1911
base_lines = split_lines(TEXT_1)
1912
source.add_lines('version-1', [], base_lines)
1913
source.add_lines('version-2', ['version-1'], base_lines + ['a\n'])
1914
# The second record should be a delta.
1915
self.assertEqual('line-delta', source._index.get_method('version-2'))
1916
data_stream = source.get_data_stream(['version-1', 'version-2'])
1918
# Insert the stream into a non-delta knit.
1919
target = self.make_test_knit(name='target', delta=False)
1920
target.insert_data_stream(data_stream)
1922
# Both versions are fulltexts in the target
1923
self.assertEqual('fulltext', target._index.get_method('version-1'))
1924
self.assertEqual('fulltext', target._index.get_method('version-2'))
1927
class DataStreamTests(KnitTests):
1929
def assertMadeStreamKnit(self, source_knit, versions, target_knit):
1930
"""Assert that a knit made from a stream is as expected."""
1931
a_stream = source_knit.get_data_stream(versions)
1932
expected_data = a_stream[2](None)
1933
a_stream = source_knit.get_data_stream(versions)
1934
a_knit = target_knit._knit_from_datastream(a_stream)
1935
self.assertEqual(source_knit.factory.__class__,
1936
a_knit.factory.__class__)
1937
self.assertIsInstance(a_knit._data._access, _StreamAccess)
1938
self.assertIsInstance(a_knit._index, _StreamIndex)
1939
self.assertEqual(a_knit._index.data_list, a_stream[1])
1940
self.assertEqual(a_knit._data._access.data, expected_data)
1941
self.assertEqual(a_knit.filename, target_knit.filename)
1942
self.assertEqual(a_knit.transport, target_knit.transport)
1943
self.assertEqual(a_knit._index, a_knit._data._access.stream_index)
1944
self.assertEqual(target_knit, a_knit._data._access.backing_knit)
1945
self.assertIsInstance(a_knit._data._access.orig_factory,
1946
source_knit.factory.__class__)
1948
def test__knit_from_data_stream_empty(self):
1949
"""Create a knit object from a datastream."""
1950
annotated = self.make_test_knit(name='source', annotate=True)
1951
plain = self.make_test_knit(name='target', annotate=False)
1952
# case 1: annotated source
1953
self.assertMadeStreamKnit(annotated, [], annotated)
1954
self.assertMadeStreamKnit(annotated, [], plain)
1955
# case 2: plain source
1956
self.assertMadeStreamKnit(plain, [], annotated)
1957
self.assertMadeStreamKnit(plain, [], plain)
1959
def test__knit_from_data_stream_unknown_format(self):
1960
annotated = self.make_test_knit(name='source', annotate=True)
1961
self.assertRaises(errors.KnitDataStreamUnknown,
1962
annotated._knit_from_datastream, ("unknown", None, None))
1974
Banana cup cake recipe
1980
- self-raising flour
1984
Banana cup cake recipe
1986
- bananas (do not use plantains!!!)
1993
Banana cup cake recipe
1996
- self-raising flour
2009
AB_MERGE_TEXT="""unchanged|Banana cup cake recipe
2014
new-b|- bananas (do not use plantains!!!)
2015
unchanged|- broken tea cups
2016
new-a|- self-raising flour
2019
AB_MERGE=[tuple(l.split('|')) for l in AB_MERGE_TEXT.splitlines(True)]
2022
def line_delta(from_lines, to_lines):
2023
"""Generate line-based delta from one text to another"""
2024
s = difflib.SequenceMatcher(None, from_lines, to_lines)
2025
for op in s.get_opcodes():
2026
if op[0] == 'equal':
2028
yield '%d,%d,%d\n' % (op[1], op[2], op[4]-op[3])
2029
for i in range(op[3], op[4]):
2033
def apply_line_delta(basis_lines, delta_lines):
2034
"""Apply a line-based perfect diff
2036
basis_lines -- text to apply the patch to
2037
delta_lines -- diff instructions and content
2039
out = basis_lines[:]
2042
while i < len(delta_lines):
2044
a, b, c = map(long, l.split(','))
2046
out[offset+a:offset+b] = delta_lines[i:i+c]
2048
offset = offset + (b - a) + c
2052
class TestWeaveToKnit(KnitTests):
2054
def test_weave_to_knit_matches(self):
2055
# check that the WeaveToKnit is_compatible function
2056
# registers True for a Weave to a Knit.
2057
w = Weave(get_scope=lambda:None)
2058
k = self.make_test_knit()
2059
self.failUnless(WeaveToKnit.is_compatible(w, k))
2060
self.failIf(WeaveToKnit.is_compatible(k, w))
2061
self.failIf(WeaveToKnit.is_compatible(w, w))
2062
self.failIf(WeaveToKnit.is_compatible(k, k))
1505
2065
class TestKnitIndex(KnitTests):
1633
2184
add_callback = self.catch_add
1635
2186
add_callback = None
1636
return _KnitGraphIndex(combined_index, lambda:True, deltas=deltas,
2187
return KnitGraphIndex(combined_index, deltas=deltas,
1637
2188
add_callback=add_callback)
1639
def test_keys(self):
1640
index = self.two_graph_index()
1641
self.assertEqual(set([('tail',), ('tip',), ('parent',), ('separate',)]),
2190
def test_get_ancestry(self):
2191
# get_ancestry is defined as eliding ghosts, not erroring.
2192
index = self.two_graph_index()
2193
self.assertEqual([], index.get_ancestry([]))
2194
self.assertEqual(['separate'], index.get_ancestry(['separate']))
2195
self.assertEqual(['tail'], index.get_ancestry(['tail']))
2196
self.assertEqual(['tail', 'parent'], index.get_ancestry(['parent']))
2197
self.assertEqual(['tail', 'parent', 'tip'], index.get_ancestry(['tip']))
2198
self.assertTrue(index.get_ancestry(['tip', 'separate']) in
2199
(['tail', 'parent', 'tip', 'separate'],
2200
['separate', 'tail', 'parent', 'tip'],
2202
# and without topo_sort
2203
self.assertEqual(set(['separate']),
2204
set(index.get_ancestry(['separate'], topo_sorted=False)))
2205
self.assertEqual(set(['tail']),
2206
set(index.get_ancestry(['tail'], topo_sorted=False)))
2207
self.assertEqual(set(['tail', 'parent']),
2208
set(index.get_ancestry(['parent'], topo_sorted=False)))
2209
self.assertEqual(set(['tail', 'parent', 'tip']),
2210
set(index.get_ancestry(['tip'], topo_sorted=False)))
2211
self.assertEqual(set(['separate', 'tail', 'parent', 'tip']),
2212
set(index.get_ancestry(['tip', 'separate'])))
2213
# asking for a ghost makes it go boom.
2214
self.assertRaises(errors.RevisionNotPresent, index.get_ancestry, ['ghost'])
2216
def test_get_ancestry_with_ghosts(self):
2217
index = self.two_graph_index()
2218
self.assertEqual([], index.get_ancestry_with_ghosts([]))
2219
self.assertEqual(['separate'], index.get_ancestry_with_ghosts(['separate']))
2220
self.assertEqual(['tail'], index.get_ancestry_with_ghosts(['tail']))
2221
self.assertTrue(index.get_ancestry_with_ghosts(['parent']) in
2222
(['tail', 'ghost', 'parent'],
2223
['ghost', 'tail', 'parent'],
2225
self.assertTrue(index.get_ancestry_with_ghosts(['tip']) in
2226
(['tail', 'ghost', 'parent', 'tip'],
2227
['ghost', 'tail', 'parent', 'tip'],
2229
self.assertTrue(index.get_ancestry_with_ghosts(['tip', 'separate']) in
2230
(['tail', 'ghost', 'parent', 'tip', 'separate'],
2231
['ghost', 'tail', 'parent', 'tip', 'separate'],
2232
['separate', 'tail', 'ghost', 'parent', 'tip'],
2233
['separate', 'ghost', 'tail', 'parent', 'tip'],
2235
# asking for a ghost makes it go boom.
2236
self.assertRaises(errors.RevisionNotPresent, index.get_ancestry_with_ghosts, ['ghost'])
2238
def test_num_versions(self):
2239
index = self.two_graph_index()
2240
self.assertEqual(4, index.num_versions())
2242
def test_get_versions(self):
2243
index = self.two_graph_index()
2244
self.assertEqual(set(['tail', 'tip', 'parent', 'separate']),
2245
set(index.get_versions()))
2247
def test_has_version(self):
2248
index = self.two_graph_index()
2249
self.assertTrue(index.has_version('tail'))
2250
self.assertFalse(index.has_version('ghost'))
1644
2252
def test_get_position(self):
1645
2253
index = self.two_graph_index()
1646
self.assertEqual((index._graph_index._indices[0], 0, 100), index.get_position(('tip',)))
1647
self.assertEqual((index._graph_index._indices[1], 100, 78), index.get_position(('parent',)))
2254
self.assertEqual((index._graph_index._indices[0], 0, 100), index.get_position('tip'))
2255
self.assertEqual((index._graph_index._indices[1], 100, 78), index.get_position('parent'))
1649
2257
def test_get_method_deltas(self):
1650
2258
index = self.two_graph_index(deltas=True)
1651
self.assertEqual('fulltext', index.get_method(('tip',)))
1652
self.assertEqual('line-delta', index.get_method(('parent',)))
2259
self.assertEqual('fulltext', index.get_method('tip'))
2260
self.assertEqual('line-delta', index.get_method('parent'))
1654
2262
def test_get_method_no_deltas(self):
1655
2263
# check that the parent-history lookup is ignored with deltas=False.
1656
2264
index = self.two_graph_index(deltas=False)
1657
self.assertEqual('fulltext', index.get_method(('tip',)))
1658
self.assertEqual('fulltext', index.get_method(('parent',)))
2265
self.assertEqual('fulltext', index.get_method('tip'))
2266
self.assertEqual('fulltext', index.get_method('parent'))
1660
2268
def test_get_options_deltas(self):
1661
2269
index = self.two_graph_index(deltas=True)
1662
self.assertEqual(['fulltext', 'no-eol'], index.get_options(('tip',)))
1663
self.assertEqual(['line-delta'], index.get_options(('parent',)))
2270
self.assertEqual(['fulltext', 'no-eol'], index.get_options('tip'))
2271
self.assertEqual(['line-delta'], index.get_options('parent'))
1665
2273
def test_get_options_no_deltas(self):
1666
2274
# check that the parent-history lookup is ignored with deltas=False.
1667
2275
index = self.two_graph_index(deltas=False)
1668
self.assertEqual(['fulltext', 'no-eol'], index.get_options(('tip',)))
1669
self.assertEqual(['fulltext'], index.get_options(('parent',)))
1671
def test_get_parent_map(self):
1672
index = self.two_graph_index()
1673
self.assertEqual({('parent',):(('tail',), ('ghost',))},
1674
index.get_parent_map([('parent',), ('ghost',)]))
2276
self.assertEqual(['fulltext', 'no-eol'], index.get_options('tip'))
2277
self.assertEqual(['fulltext'], index.get_options('parent'))
2279
def test_get_parents_with_ghosts(self):
2280
index = self.two_graph_index()
2281
self.assertEqual(('tail', 'ghost'), index.get_parents_with_ghosts('parent'))
2282
# and errors on ghosts.
2283
self.assertRaises(errors.RevisionNotPresent,
2284
index.get_parents_with_ghosts, 'ghost')
2286
def test_check_versions_present(self):
2287
# ghosts should not be considered present
2288
index = self.two_graph_index()
2289
self.assertRaises(RevisionNotPresent, index.check_versions_present,
2291
self.assertRaises(RevisionNotPresent, index.check_versions_present,
2293
index.check_versions_present(['tail', 'separate'])
1676
2295
def catch_add(self, entries):
1677
2296
self.caught_entries.append(entries)
1679
2298
def test_add_no_callback_errors(self):
1680
2299
index = self.two_graph_index()
1681
self.assertRaises(errors.ReadOnlyError, index.add_records,
1682
[(('new',), 'fulltext,no-eol', (None, 50, 60), ['separate'])])
2300
self.assertRaises(errors.ReadOnlyError, index.add_version,
2301
'new', 'fulltext,no-eol', (None, 50, 60), ['separate'])
1684
2303
def test_add_version_smoke(self):
1685
2304
index = self.two_graph_index(catch_adds=True)
1686
index.add_records([(('new',), 'fulltext,no-eol', (None, 50, 60),
2305
index.add_version('new', 'fulltext,no-eol', (None, 50, 60), ['separate'])
1688
2306
self.assertEqual([[(('new', ), 'N50 60', ((('separate',),),))]],
1689
2307
self.caught_entries)
1691
2309
def test_add_version_delta_not_delta_index(self):
1692
2310
index = self.two_graph_index(catch_adds=True)
1693
self.assertRaises(errors.KnitCorrupt, index.add_records,
1694
[(('new',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
2311
self.assertRaises(errors.KnitCorrupt, index.add_version,
2312
'new', 'no-eol,line-delta', (None, 0, 100), ['parent'])
1695
2313
self.assertEqual([], self.caught_entries)
1697
2315
def test_add_version_same_dup(self):
1698
2316
index = self.two_graph_index(catch_adds=True)
1699
2317
# options can be spelt two different ways
1700
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1701
index.add_records([(('tip',), 'no-eol,fulltext', (None, 0, 100), [('parent',)])])
1702
# position/length are ignored (because each pack could have fulltext or
1703
# delta, and be at a different position.
1704
index.add_records([(('tip',), 'fulltext,no-eol', (None, 50, 100),
1706
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000),
1708
# but neither should have added data:
1709
self.assertEqual([[], [], [], []], self.caught_entries)
2318
index.add_version('tip', 'fulltext,no-eol', (None, 0, 100), ['parent'])
2319
index.add_version('tip', 'no-eol,fulltext', (None, 0, 100), ['parent'])
2320
# but neither should have added data.
2321
self.assertEqual([[], []], self.caught_entries)
1711
2323
def test_add_version_different_dup(self):
1712
2324
index = self.two_graph_index(deltas=True, catch_adds=True)
1713
2325
# change options
1714
self.assertRaises(errors.KnitCorrupt, index.add_records,
1715
[(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1716
self.assertRaises(errors.KnitCorrupt, index.add_records,
1717
[(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
2326
self.assertRaises(errors.KnitCorrupt, index.add_version,
2327
'tip', 'no-eol,line-delta', (None, 0, 100), ['parent'])
2328
self.assertRaises(errors.KnitCorrupt, index.add_version,
2329
'tip', 'line-delta,no-eol', (None, 0, 100), ['parent'])
2330
self.assertRaises(errors.KnitCorrupt, index.add_version,
2331
'tip', 'fulltext', (None, 0, 100), ['parent'])
2333
self.assertRaises(errors.KnitCorrupt, index.add_version,
2334
'tip', 'fulltext,no-eol', (None, 50, 100), ['parent'])
2335
self.assertRaises(errors.KnitCorrupt, index.add_version,
2336
'tip', 'fulltext,no-eol', (None, 0, 1000), ['parent'])
1719
self.assertRaises(errors.KnitCorrupt, index.add_records,
1720
[(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
2338
self.assertRaises(errors.KnitCorrupt, index.add_version,
2339
'tip', 'fulltext,no-eol', (None, 0, 100), [])
1721
2340
self.assertEqual([], self.caught_entries)
1723
2342
def test_add_versions_nodeltas(self):
1724
2343
index = self.two_graph_index(catch_adds=True)
1726
(('new',), 'fulltext,no-eol', (None, 50, 60), [('separate',)]),
1727
(('new2',), 'fulltext', (None, 0, 6), [('new',)]),
2344
index.add_versions([
2345
('new', 'fulltext,no-eol', (None, 50, 60), ['separate']),
2346
('new2', 'fulltext', (None, 0, 6), ['new']),
1729
2348
self.assertEqual([(('new', ), 'N50 60', ((('separate',),),)),
1730
2349
(('new2', ), ' 0 6', ((('new',),),))],
2024
2575
def test_add_versions_delta_not_delta_index(self):
2025
2576
index = self.two_graph_index(catch_adds=True)
2026
self.assertRaises(errors.KnitCorrupt, index.add_records,
2027
[(('new',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
2577
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2578
[('new', 'no-eol,line-delta', (None, 0, 100), ['parent'])])
2028
2579
self.assertEqual([], self.caught_entries)
2030
2581
def test_add_versions_parents_not_parents_index(self):
2031
2582
index = self.two_graph_index(catch_adds=True)
2032
self.assertRaises(errors.KnitCorrupt, index.add_records,
2033
[(('new',), 'no-eol,fulltext', (None, 0, 100), [('parent',)])])
2583
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2584
[('new', 'no-eol,fulltext', (None, 0, 100), ['parent'])])
2034
2585
self.assertEqual([], self.caught_entries)
2036
2587
def test_add_versions_random_id_accepted(self):
2037
2588
index = self.two_graph_index(catch_adds=True)
2038
index.add_records([], random_id=True)
2589
index.add_versions([], random_id=True)
2040
2591
def test_add_versions_same_dup(self):
2041
2592
index = self.two_graph_index(catch_adds=True)
2042
2593
# options can be spelt two different ways
2043
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
2044
index.add_records([(('tip',), 'no-eol,fulltext', (None, 0, 100), [])])
2045
# position/length are ignored (because each pack could have fulltext or
2046
# delta, and be at a different position.
2047
index.add_records([(('tip',), 'fulltext,no-eol', (None, 50, 100), [])])
2048
index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
2594
index.add_versions([('tip', 'fulltext,no-eol', (None, 0, 100), [])])
2595
index.add_versions([('tip', 'no-eol,fulltext', (None, 0, 100), [])])
2049
2596
# but neither should have added data.
2050
self.assertEqual([[], [], [], []], self.caught_entries)
2597
self.assertEqual([[], []], self.caught_entries)
2052
2599
def test_add_versions_different_dup(self):
2053
2600
index = self.two_graph_index(catch_adds=True)
2054
2601
# change options
2055
self.assertRaises(errors.KnitCorrupt, index.add_records,
2056
[(('tip',), 'no-eol,line-delta', (None, 0, 100), [])])
2057
self.assertRaises(errors.KnitCorrupt, index.add_records,
2058
[(('tip',), 'line-delta,no-eol', (None, 0, 100), [])])
2059
self.assertRaises(errors.KnitCorrupt, index.add_records,
2060
[(('tip',), 'fulltext', (None, 0, 100), [])])
2602
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2603
[('tip', 'no-eol,line-delta', (None, 0, 100), [])])
2604
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2605
[('tip', 'line-delta,no-eol', (None, 0, 100), [])])
2606
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2607
[('tip', 'fulltext', (None, 0, 100), [])])
2609
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2610
[('tip', 'fulltext,no-eol', (None, 50, 100), [])])
2611
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2612
[('tip', 'fulltext,no-eol', (None, 0, 1000), [])])
2062
self.assertRaises(errors.KnitCorrupt, index.add_records,
2063
[(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
2614
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2615
[('tip', 'fulltext,no-eol', (None, 0, 100), ['parent'])])
2064
2616
# change options in the second record
2065
self.assertRaises(errors.KnitCorrupt, index.add_records,
2066
[(('tip',), 'fulltext,no-eol', (None, 0, 100), []),
2067
(('tip',), 'no-eol,line-delta', (None, 0, 100), [])])
2617
self.assertRaises(errors.KnitCorrupt, index.add_versions,
2618
[('tip', 'fulltext,no-eol', (None, 0, 100), []),
2619
('tip', 'no-eol,line-delta', (None, 0, 100), [])])
2068
2620
self.assertEqual([], self.caught_entries)
2071
class TestKnitVersionedFiles(KnitTests):
2073
def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
2074
positions, _min_buffer_size=None):
2075
kvf = self.make_test_knit()
2076
if _min_buffer_size is None:
2077
_min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
2078
self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
2079
non_local_keys, positions,
2080
_min_buffer_size=_min_buffer_size))
2082
def assertSplitByPrefix(self, expected_map, expected_prefix_order,
2084
split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
2085
self.assertEqual(expected_map, split)
2086
self.assertEqual(expected_prefix_order, prefix_order)
2088
def test__group_keys_for_io(self):
2089
ft_detail = ('fulltext', False)
2090
ld_detail = ('line-delta', False)
2098
f_a: (ft_detail, (f_a, 0, 100), None),
2099
f_b: (ld_detail, (f_b, 100, 21), f_a),
2100
f_c: (ld_detail, (f_c, 180, 15), f_b),
2101
g_a: (ft_detail, (g_a, 121, 35), None),
2102
g_b: (ld_detail, (g_b, 156, 12), g_a),
2103
g_c: (ld_detail, (g_c, 195, 13), g_a),
2105
self.assertGroupKeysForIo([([f_a], set())],
2106
[f_a], [], positions)
2107
self.assertGroupKeysForIo([([f_a], set([f_a]))],
2108
[f_a], [f_a], positions)
2109
self.assertGroupKeysForIo([([f_a, f_b], set([]))],
2110
[f_a, f_b], [], positions)
2111
self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
2112
[f_a, f_b], [f_b], positions)
2113
self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
2114
[f_a, g_a, f_b, g_b], [], positions)
2115
self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
2116
[f_a, g_a, f_b, g_b], [], positions,
2117
_min_buffer_size=150)
2118
self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
2119
[f_a, g_a, f_b, g_b], [], positions,
2120
_min_buffer_size=100)
2121
self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
2122
[f_c, g_b], [], positions,
2123
_min_buffer_size=125)
2124
self.assertGroupKeysForIo([([g_b, f_c], set())],
2125
[g_b, f_c], [], positions,
2126
_min_buffer_size=125)
2128
def test__split_by_prefix(self):
2129
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2130
'g': [('g', 'b'), ('g', 'a')],
2132
[('f', 'a'), ('g', 'b'),
2133
('g', 'a'), ('f', 'b')])
2135
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2136
'g': [('g', 'b'), ('g', 'a')],
2138
[('f', 'a'), ('f', 'b'),
2139
('g', 'b'), ('g', 'a')])
2141
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2142
'g': [('g', 'b'), ('g', 'a')],
2144
[('f', 'a'), ('f', 'b'),
2145
('g', 'b'), ('g', 'a')])
2147
self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
2148
'g': [('g', 'b'), ('g', 'a')],
2149
'': [('a',), ('b',)]
2151
[('f', 'a'), ('g', 'b'),
2153
('g', 'a'), ('f', 'b')])
2156
class TestStacking(KnitTests):
2158
def get_basis_and_test_knit(self):
2159
basis = self.make_test_knit(name='basis')
2160
basis = RecordingVersionedFilesDecorator(basis)
2161
test = self.make_test_knit(name='test')
2162
test.add_fallback_versioned_files(basis)
2165
def test_add_fallback_versioned_files(self):
2166
basis = self.make_test_knit(name='basis')
2167
test = self.make_test_knit(name='test')
2168
# It must not error; other tests test that the fallback is referred to
2169
# when accessing data.
2170
test.add_fallback_versioned_files(basis)
2172
def test_add_lines(self):
2173
# lines added to the test are not added to the basis
2174
basis, test = self.get_basis_and_test_knit()
2176
key_basis = ('bar',)
2177
key_cross_border = ('quux',)
2178
key_delta = ('zaphod',)
2179
test.add_lines(key, (), ['foo\n'])
2180
self.assertEqual({}, basis.get_parent_map([key]))
2181
# lines added to the test that reference across the stack do a
2183
basis.add_lines(key_basis, (), ['foo\n'])
2185
test.add_lines(key_cross_border, (key_basis,), ['foo\n'])
2186
self.assertEqual('fulltext', test._index.get_method(key_cross_border))
2187
# we don't even need to look at the basis to see that this should be
2188
# stored as a fulltext
2189
self.assertEqual([], basis.calls)
2190
# Subsequent adds do delta.
2192
test.add_lines(key_delta, (key_cross_border,), ['foo\n'])
2193
self.assertEqual('line-delta', test._index.get_method(key_delta))
2194
self.assertEqual([], basis.calls)
2196
def test_annotate(self):
2197
# annotations from the test knit are answered without asking the basis
2198
basis, test = self.get_basis_and_test_knit()
2200
key_basis = ('bar',)
2201
key_missing = ('missing',)
2202
test.add_lines(key, (), ['foo\n'])
2203
details = test.annotate(key)
2204
self.assertEqual([(key, 'foo\n')], details)
2205
self.assertEqual([], basis.calls)
2206
# But texts that are not in the test knit are looked for in the basis
2208
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2210
details = test.annotate(key_basis)
2211
self.assertEqual([(key_basis, 'foo\n'), (key_basis, 'bar\n')], details)
2212
# Not optimised to date:
2213
# self.assertEqual([("annotate", key_basis)], basis.calls)
2214
self.assertEqual([('get_parent_map', set([key_basis])),
2215
('get_parent_map', set([key_basis])),
2216
('get_record_stream', [key_basis], 'topological', True)],
2219
def test_check(self):
2220
# At the moment checking a stacked knit does implicitly check the
2222
basis, test = self.get_basis_and_test_knit()
2225
def test_get_parent_map(self):
2226
# parents in the test knit are answered without asking the basis
2227
basis, test = self.get_basis_and_test_knit()
2229
key_basis = ('bar',)
2230
key_missing = ('missing',)
2231
test.add_lines(key, (), [])
2232
parent_map = test.get_parent_map([key])
2233
self.assertEqual({key: ()}, parent_map)
2234
self.assertEqual([], basis.calls)
2235
# But parents that are not in the test knit are looked for in the basis
2236
basis.add_lines(key_basis, (), [])
2238
parent_map = test.get_parent_map([key, key_basis, key_missing])
2239
self.assertEqual({key: (),
2240
key_basis: ()}, parent_map)
2241
self.assertEqual([("get_parent_map", set([key_basis, key_missing]))],
2244
def test_get_record_stream_unordered_fulltexts(self):
2245
# records from the test knit are answered without asking the basis:
2246
basis, test = self.get_basis_and_test_knit()
2248
key_basis = ('bar',)
2249
key_missing = ('missing',)
2250
test.add_lines(key, (), ['foo\n'])
2251
records = list(test.get_record_stream([key], 'unordered', True))
2252
self.assertEqual(1, len(records))
2253
self.assertEqual([], basis.calls)
2254
# Missing (from test knit) objects are retrieved from the basis:
2255
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2257
records = list(test.get_record_stream([key_basis, key_missing],
2259
self.assertEqual(2, len(records))
2260
calls = list(basis.calls)
2261
for record in records:
2262
self.assertSubset([record.key], (key_basis, key_missing))
2263
if record.key == key_missing:
2264
self.assertIsInstance(record, AbsentContentFactory)
2266
reference = list(basis.get_record_stream([key_basis],
2267
'unordered', True))[0]
2268
self.assertEqual(reference.key, record.key)
2269
self.assertEqual(reference.sha1, record.sha1)
2270
self.assertEqual(reference.storage_kind, record.storage_kind)
2271
self.assertEqual(reference.get_bytes_as(reference.storage_kind),
2272
record.get_bytes_as(record.storage_kind))
2273
self.assertEqual(reference.get_bytes_as('fulltext'),
2274
record.get_bytes_as('fulltext'))
2275
# It's not strictly minimal, but it seems reasonable for now for it to
2276
# ask which fallbacks have which parents.
2278
("get_parent_map", set([key_basis, key_missing])),
2279
("get_record_stream", [key_basis], 'unordered', True)],
2282
def test_get_record_stream_ordered_fulltexts(self):
2283
# ordering is preserved down into the fallback store.
2284
basis, test = self.get_basis_and_test_knit()
2286
key_basis = ('bar',)
2287
key_basis_2 = ('quux',)
2288
key_missing = ('missing',)
2289
test.add_lines(key, (key_basis,), ['foo\n'])
2290
# Missing (from test knit) objects are retrieved from the basis:
2291
basis.add_lines(key_basis, (key_basis_2,), ['foo\n', 'bar\n'])
2292
basis.add_lines(key_basis_2, (), ['quux\n'])
2294
# ask for in non-topological order
2295
records = list(test.get_record_stream(
2296
[key, key_basis, key_missing, key_basis_2], 'topological', True))
2297
self.assertEqual(4, len(records))
2299
for record in records:
2300
self.assertSubset([record.key],
2301
(key_basis, key_missing, key_basis_2, key))
2302
if record.key == key_missing:
2303
self.assertIsInstance(record, AbsentContentFactory)
2305
results.append((record.key, record.sha1, record.storage_kind,
2306
record.get_bytes_as('fulltext')))
2307
calls = list(basis.calls)
2308
order = [record[0] for record in results]
2309
self.assertEqual([key_basis_2, key_basis, key], order)
2310
for result in results:
2311
if result[0] == key:
2315
record = source.get_record_stream([result[0]], 'unordered',
2317
self.assertEqual(record.key, result[0])
2318
self.assertEqual(record.sha1, result[1])
2319
# We used to check that the storage kind matched, but actually it
2320
# depends on whether it was sourced from the basis, or in a single
2321
# group, because asking for full texts returns proxy objects to a
2322
# _ContentMapGenerator object; so checking the kind is unneeded.
2323
self.assertEqual(record.get_bytes_as('fulltext'), result[3])
2324
# It's not strictly minimal, but it seems reasonable for now for it to
2325
# ask which fallbacks have which parents.
2327
("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2328
# topological is requested from the fallback, because that is what
2329
# was requested at the top level.
2330
("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
2333
def test_get_record_stream_unordered_deltas(self):
2334
# records from the test knit are answered without asking the basis:
2335
basis, test = self.get_basis_and_test_knit()
2337
key_basis = ('bar',)
2338
key_missing = ('missing',)
2339
test.add_lines(key, (), ['foo\n'])
2340
records = list(test.get_record_stream([key], 'unordered', False))
2341
self.assertEqual(1, len(records))
2342
self.assertEqual([], basis.calls)
2343
# Missing (from test knit) objects are retrieved from the basis:
2344
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2346
records = list(test.get_record_stream([key_basis, key_missing],
2347
'unordered', False))
2348
self.assertEqual(2, len(records))
2349
calls = list(basis.calls)
2350
for record in records:
2351
self.assertSubset([record.key], (key_basis, key_missing))
2352
if record.key == key_missing:
2353
self.assertIsInstance(record, AbsentContentFactory)
2355
reference = list(basis.get_record_stream([key_basis],
2356
'unordered', False))[0]
2357
self.assertEqual(reference.key, record.key)
2358
self.assertEqual(reference.sha1, record.sha1)
2359
self.assertEqual(reference.storage_kind, record.storage_kind)
2360
self.assertEqual(reference.get_bytes_as(reference.storage_kind),
2361
record.get_bytes_as(record.storage_kind))
2362
# It's not strictly minimal, but it seems reasonable for now for it to
2363
# ask which fallbacks have which parents.
2365
("get_parent_map", set([key_basis, key_missing])),
2366
("get_record_stream", [key_basis], 'unordered', False)],
2369
def test_get_record_stream_ordered_deltas(self):
2370
# ordering is preserved down into the fallback store.
2371
basis, test = self.get_basis_and_test_knit()
2373
key_basis = ('bar',)
2374
key_basis_2 = ('quux',)
2375
key_missing = ('missing',)
2376
test.add_lines(key, (key_basis,), ['foo\n'])
2377
# Missing (from test knit) objects are retrieved from the basis:
2378
basis.add_lines(key_basis, (key_basis_2,), ['foo\n', 'bar\n'])
2379
basis.add_lines(key_basis_2, (), ['quux\n'])
2381
# ask for in non-topological order
2382
records = list(test.get_record_stream(
2383
[key, key_basis, key_missing, key_basis_2], 'topological', False))
2384
self.assertEqual(4, len(records))
2386
for record in records:
2387
self.assertSubset([record.key],
2388
(key_basis, key_missing, key_basis_2, key))
2389
if record.key == key_missing:
2390
self.assertIsInstance(record, AbsentContentFactory)
2392
results.append((record.key, record.sha1, record.storage_kind,
2393
record.get_bytes_as(record.storage_kind)))
2394
calls = list(basis.calls)
2395
order = [record[0] for record in results]
2396
self.assertEqual([key_basis_2, key_basis, key], order)
2397
for result in results:
2398
if result[0] == key:
2402
record = source.get_record_stream([result[0]], 'unordered',
2404
self.assertEqual(record.key, result[0])
2405
self.assertEqual(record.sha1, result[1])
2406
self.assertEqual(record.storage_kind, result[2])
2407
self.assertEqual(record.get_bytes_as(record.storage_kind), result[3])
2408
# It's not strictly minimal, but it seems reasonable for now for it to
2409
# ask which fallbacks have which parents.
2411
("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2412
("get_record_stream", [key_basis_2, key_basis], 'topological', False)],
2415
def test_get_sha1s(self):
2416
# sha1's in the test knit are answered without asking the basis
2417
basis, test = self.get_basis_and_test_knit()
2419
key_basis = ('bar',)
2420
key_missing = ('missing',)
2421
test.add_lines(key, (), ['foo\n'])
2422
key_sha1sum = osutils.sha('foo\n').hexdigest()
2423
sha1s = test.get_sha1s([key])
2424
self.assertEqual({key: key_sha1sum}, sha1s)
2425
self.assertEqual([], basis.calls)
2426
# But texts that are not in the test knit are looked for in the basis
2427
# directly (rather than via text reconstruction) so that remote servers
2428
# etc don't have to answer with full content.
2429
basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2430
basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2432
sha1s = test.get_sha1s([key, key_missing, key_basis])
2433
self.assertEqual({key: key_sha1sum,
2434
key_basis: basis_sha1sum}, sha1s)
2435
self.assertEqual([("get_sha1s", set([key_basis, key_missing]))],
2438
def test_insert_record_stream(self):
2439
# records are inserted as normal; insert_record_stream builds on
2440
# add_lines, so a smoke test should be all that's needed:
2442
key_basis = ('bar',)
2443
key_delta = ('zaphod',)
2444
basis, test = self.get_basis_and_test_knit()
2445
source = self.make_test_knit(name='source')
2446
basis.add_lines(key_basis, (), ['foo\n'])
2448
source.add_lines(key_basis, (), ['foo\n'])
2449
source.add_lines(key_delta, (key_basis,), ['bar\n'])
2450
stream = source.get_record_stream([key_delta], 'unordered', False)
2451
test.insert_record_stream(stream)
2452
# XXX: this does somewhat too many calls in making sure of whether it
2453
# has to recreate the full text.
2454
self.assertEqual([("get_parent_map", set([key_basis])),
2455
('get_parent_map', set([key_basis])),
2456
('get_record_stream', [key_basis], 'unordered', True)],
2458
self.assertEqual({key_delta:(key_basis,)},
2459
test.get_parent_map([key_delta]))
2460
self.assertEqual('bar\n', test.get_record_stream([key_delta],
2461
'unordered', True).next().get_bytes_as('fulltext'))
2463
def test_iter_lines_added_or_present_in_keys(self):
2464
# Lines from the basis are returned, and lines for a given key are only
2468
# all sources are asked for keys:
2469
basis, test = self.get_basis_and_test_knit()
2470
basis.add_lines(key1, (), ["foo"])
2472
lines = list(test.iter_lines_added_or_present_in_keys([key1]))
2473
self.assertEqual([("foo\n", key1)], lines)
2474
self.assertEqual([("iter_lines_added_or_present_in_keys", set([key1]))],
2476
# keys in both are not duplicated:
2477
test.add_lines(key2, (), ["bar\n"])
2478
basis.add_lines(key2, (), ["bar\n"])
2480
lines = list(test.iter_lines_added_or_present_in_keys([key2]))
2481
self.assertEqual([("bar\n", key2)], lines)
2482
self.assertEqual([], basis.calls)
2484
def test_keys(self):
2487
# all sources are asked for keys:
2488
basis, test = self.get_basis_and_test_knit()
2490
self.assertEqual(set(), set(keys))
2491
self.assertEqual([("keys",)], basis.calls)
2492
# keys from a basis are returned:
2493
basis.add_lines(key1, (), [])
2496
self.assertEqual(set([key1]), set(keys))
2497
self.assertEqual([("keys",)], basis.calls)
2498
# keys in both are not duplicated:
2499
test.add_lines(key2, (), [])
2500
basis.add_lines(key2, (), [])
2503
self.assertEqual(2, len(keys))
2504
self.assertEqual(set([key1, key2]), set(keys))
2505
self.assertEqual([("keys",)], basis.calls)
2507
def test_add_mpdiffs(self):
2508
# records are inserted as normal; add_mpdiff builds on
2509
# add_lines, so a smoke test should be all that's needed:
2511
key_basis = ('bar',)
2512
key_delta = ('zaphod',)
2513
basis, test = self.get_basis_and_test_knit()
2514
source = self.make_test_knit(name='source')
2515
basis.add_lines(key_basis, (), ['foo\n'])
2517
source.add_lines(key_basis, (), ['foo\n'])
2518
source.add_lines(key_delta, (key_basis,), ['bar\n'])
2519
diffs = source.make_mpdiffs([key_delta])
2520
test.add_mpdiffs([(key_delta, (key_basis,),
2521
source.get_sha1s([key_delta])[key_delta], diffs[0])])
2522
self.assertEqual([("get_parent_map", set([key_basis])),
2523
('get_record_stream', [key_basis], 'unordered', True),],
2525
self.assertEqual({key_delta:(key_basis,)},
2526
test.get_parent_map([key_delta]))
2527
self.assertEqual('bar\n', test.get_record_stream([key_delta],
2528
'unordered', True).next().get_bytes_as('fulltext'))
2530
def test_make_mpdiffs(self):
2531
# Generating an mpdiff across a stacking boundary should detect parent
2535
key_right = ('zaphod',)
2536
basis, test = self.get_basis_and_test_knit()
2537
basis.add_lines(key_left, (), ['bar\n'])
2538
basis.add_lines(key_right, (), ['zaphod\n'])
2540
test.add_lines(key, (key_left, key_right),
2541
['bar\n', 'foo\n', 'zaphod\n'])
2542
diffs = test.make_mpdiffs([key])
2544
multiparent.MultiParent([multiparent.ParentText(0, 0, 0, 1),
2545
multiparent.NewText(['foo\n']),
2546
multiparent.ParentText(1, 0, 2, 1)])],
2548
self.assertEqual(3, len(basis.calls))
2550
("get_parent_map", set([key_left, key_right])),
2551
("get_parent_map", set([key_left, key_right])),
2554
last_call = basis.calls[-1]
2555
self.assertEqual('get_record_stream', last_call[0])
2556
self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2557
self.assertEqual('topological', last_call[2])
2558
self.assertEqual(True, last_call[3])
2561
class TestNetworkBehaviour(KnitTests):
2562
"""Tests for getting data out of/into knits over the network."""
2564
def test_include_delta_closure_generates_a_knit_delta_closure(self):
2565
vf = self.make_test_knit(name='test')
2566
# put in three texts, giving ft, delta, delta
2567
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2568
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2569
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2570
# But heuristics could interfere, so check what happened:
2571
self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
2572
[record.storage_kind for record in
2573
vf.get_record_stream([('base',), ('d1',), ('d2',)],
2574
'topological', False)])
2575
# generate a stream of just the deltas include_delta_closure=True,
2576
# serialise to the network, and check that we get a delta closure on the wire.
2577
stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
2578
netb = [record.get_bytes_as(record.storage_kind) for record in stream]
2579
# The first bytes should be a memo from _ContentMapGenerator, and the
2580
# second bytes should be empty (because its a API proxy not something
2581
# for wire serialisation.
2582
self.assertEqual('', netb[1])
2584
kind, line_end = network_bytes_to_kind_and_offset(bytes)
2585
self.assertEqual('knit-delta-closure', kind)
2588
class TestContentMapGenerator(KnitTests):
2589
"""Tests for ContentMapGenerator"""
2591
def test_get_record_stream_gives_records(self):
2592
vf = self.make_test_knit(name='test')
2593
# put in three texts, giving ft, delta, delta
2594
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2595
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2596
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2597
keys = [('d1',), ('d2',)]
2598
generator = _VFContentMapGenerator(vf, keys,
2599
global_map=vf.get_parent_map(keys))
2600
for record in generator.get_record_stream():
2601
if record.key == ('d1',):
2602
self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
2604
self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
2606
def test_get_record_stream_kinds_are_raw(self):
2607
vf = self.make_test_knit(name='test')
2608
# put in three texts, giving ft, delta, delta
2609
vf.add_lines(('base',), (), ['base\n', 'content\n'])
2610
vf.add_lines(('d1',), (('base',),), ['d1\n'])
2611
vf.add_lines(('d2',), (('d1',),), ['d2\n'])
2612
keys = [('base',), ('d1',), ('d2',)]
2613
generator = _VFContentMapGenerator(vf, keys,
2614
global_map=vf.get_parent_map(keys))
2615
kinds = {('base',): 'knit-delta-closure',
2616
('d1',): 'knit-delta-closure-ref',
2617
('d2',): 'knit-delta-closure-ref',
2619
for record in generator.get_record_stream():
2620
self.assertEqual(kinds[record.key], record.storage_kind)
2622
class TestPackKnits(KnitTests):
2623
"""Tests that use a _PackAccess and KnitGraphIndex."""
2625
def test_get_data_stream_packs_ignores_pack_overhead(self):
2626
# Packs have an encoding overhead that should not be included in the
2627
# 'size' field of a data stream, because it is not returned by the
2628
# raw_reading functions - it is why index_memo's are opaque, and
2629
# get_data_stream was abusing this.
2630
packname = 'test.pack'
2631
transport = self.get_transport()
2632
def write_data(bytes):
2633
transport.append_bytes(packname, bytes)
2634
writer = pack.ContainerWriter(write_data)
2636
index = InMemoryGraphIndex(2)
2637
knit_index = KnitGraphIndex(index, add_callback=index.add_nodes,
2639
indices = {index:(transport, packname)}
2640
access = _PackAccess(indices, writer=(writer, index))
2641
k = KnitVersionedFile('test', get_transport('.'),
2642
delta=True, create=True, index=knit_index, access_method=access)
2643
# insert something into the knit
2644
k.add_lines('text-1', [], ["foo\n"])
2645
# get a data stream for it
2646
stream = k.get_data_stream(['text-1'])
2647
# if the stream has been incorrectly assembled, we will get a short read
2648
# reading from the stream (as streams have no trailer)
2649
expected_length = stream[1][0][2]
2650
# we use -1 to do the read, so that if a trailer is added this test
2651
# will fail and we'll adjust it to handle that case correctly, rather
2652
# than allowing an over-read that is bogus.
2653
self.assertEqual(expected_length, len(stream[2](-1)))
2656
class Test_StreamIndex(KnitTests):
2658
def get_index(self, knit, stream):
2659
"""Get a _StreamIndex from knit and stream."""
2660
return knit._knit_from_datastream(stream)._index
2662
def assertIndexVersions(self, knit, versions):
2663
"""Check that the _StreamIndex versions are those of the stream."""
2664
index = self.get_index(knit, knit.get_data_stream(versions))
2665
self.assertEqual(set(index.get_versions()), set(versions))
2666
# check we didn't get duplicates
2667
self.assertEqual(len(index.get_versions()), len(versions))
2669
def assertIndexAncestry(self, knit, ancestry_versions, versions, result):
2670
"""Check the result of a get_ancestry call on knit."""
2671
index = self.get_index(knit, knit.get_data_stream(versions))
2674
set(index.get_ancestry(ancestry_versions, False)))
2676
def assertGetMethod(self, knit, versions, version, result):
2677
index = self.get_index(knit, knit.get_data_stream(versions))
2678
self.assertEqual(result, index.get_method(version))
2680
def assertGetOptions(self, knit, version, options):
2681
index = self.get_index(knit, knit.get_data_stream(version))
2682
self.assertEqual(options, index.get_options(version))
2684
def assertGetPosition(self, knit, versions, version, result):
2685
index = self.get_index(knit, knit.get_data_stream(versions))
2686
if result[1] is None:
2687
result = (result[0], index, result[2], result[3])
2688
self.assertEqual(result, index.get_position(version))
2690
def assertGetParentsWithGhosts(self, knit, versions, version, parents):
2691
index = self.get_index(knit, knit.get_data_stream(versions))
2692
self.assertEqual(parents, index.get_parents_with_ghosts(version))
2694
def make_knit_with_4_versions_2_dags(self):
2695
knit = self.make_test_knit()
2696
knit.add_lines('a', [], ["foo"])
2697
knit.add_lines('b', [], [])
2698
knit.add_lines('c', ['b', 'a'], [])
2699
knit.add_lines_with_ghosts('d', ['e', 'f'], [])
2702
def test_versions(self):
2703
"""The versions of a StreamIndex are those of the datastream."""
2704
knit = self.make_knit_with_4_versions_2_dags()
2705
# ask for most permutations, which catches bugs like falling back to the
2706
# target knit, or showing ghosts, etc.
2707
self.assertIndexVersions(knit, [])
2708
self.assertIndexVersions(knit, ['a'])
2709
self.assertIndexVersions(knit, ['b'])
2710
self.assertIndexVersions(knit, ['c'])
2711
self.assertIndexVersions(knit, ['d'])
2712
self.assertIndexVersions(knit, ['a', 'b'])
2713
self.assertIndexVersions(knit, ['b', 'c'])
2714
self.assertIndexVersions(knit, ['a', 'c'])
2715
self.assertIndexVersions(knit, ['a', 'b', 'c'])
2716
self.assertIndexVersions(knit, ['a', 'b', 'c', 'd'])
2718
def test_construct(self):
2719
"""Constructing a StreamIndex generates index data."""
2720
data_list = [('text-a', ['fulltext'], 127, []),
2721
('text-b', ['option'], 128, ['text-c'])]
2722
index = _StreamIndex(data_list, None)
2723
self.assertEqual({'text-a':(['fulltext'], (0, 127), []),
2724
'text-b':(['option'], (127, 127 + 128), ['text-c'])},
2727
def test_get_ancestry(self):
2728
knit = self.make_knit_with_4_versions_2_dags()
2729
self.assertIndexAncestry(knit, ['a'], ['a'], ['a'])
2730
self.assertIndexAncestry(knit, ['b'], ['b'], ['b'])
2731
self.assertIndexAncestry(knit, ['c'], ['c'], ['c'])
2732
self.assertIndexAncestry(knit, ['c'], ['a', 'b', 'c'],
2733
set(['a', 'b', 'c']))
2734
self.assertIndexAncestry(knit, ['c', 'd'], ['a', 'b', 'c', 'd'],
2735
set(['a', 'b', 'c', 'd']))
2737
def test_get_method(self):
2738
knit = self.make_knit_with_4_versions_2_dags()
2739
self.assertGetMethod(knit, ['a'], 'a', 'fulltext')
2740
self.assertGetMethod(knit, ['c'], 'c', 'line-delta')
2741
# get_method on a basis that is not in the datastream (but in the
2742
# backing knit) returns 'fulltext', because thats what we'll create as
2744
self.assertGetMethod(knit, ['c'], 'b', 'fulltext')
2746
def test_get_options(self):
2747
knit = self.make_knit_with_4_versions_2_dags()
2748
self.assertGetOptions(knit, 'a', ['no-eol', 'fulltext'])
2749
self.assertGetOptions(knit, 'c', ['line-delta'])
2751
def test_get_parents_with_ghosts(self):
2752
knit = self.make_knit_with_4_versions_2_dags()
2753
self.assertGetParentsWithGhosts(knit, ['a'], 'a', ())
2754
self.assertGetParentsWithGhosts(knit, ['c'], 'c', ('b', 'a'))
2755
self.assertGetParentsWithGhosts(knit, ['d'], 'd', ('e', 'f'))
2757
def test_get_position(self):
2758
knit = self.make_knit_with_4_versions_2_dags()
2759
# get_position returns (thunk_flag, index(can be None), start, end) for
2760
# _StreamAccess to use.
2761
self.assertGetPosition(knit, ['a'], 'a', (False, None, 0, 78))
2762
self.assertGetPosition(knit, ['a', 'c'], 'c', (False, None, 78, 156))
2763
# get_position on a text that is not in the datastream (but in the
2764
# backing knit) returns (True, 'versionid', None, None) - and then the
2765
# access object can construct the relevant data as needed.
2766
self.assertGetPosition(knit, ['a', 'c'], 'b', (True, 'b', None, None))
2769
class Test_StreamAccess(KnitTests):
2771
def get_index_access(self, knit, stream):
2772
"""Get a _StreamAccess from knit and stream."""
2773
knit = knit._knit_from_datastream(stream)
2774
return knit._index, knit._data._access
2776
def assertGetRawRecords(self, knit, versions):
2777
index, access = self.get_index_access(knit,
2778
knit.get_data_stream(versions))
2779
# check that every version asked for can be obtained from the resulting
2783
for version in versions:
2784
memos.append(knit._index.get_position(version))
2786
for version, data in zip(
2787
versions, knit._data._access.get_raw_records(memos)):
2788
original[version] = data
2790
for version in versions:
2791
memos.append(index.get_position(version))
2793
for version, data in zip(versions, access.get_raw_records(memos)):
2794
streamed[version] = data
2795
self.assertEqual(original, streamed)
2797
for version in versions:
2798
data = list(access.get_raw_records(
2799
[index.get_position(version)]))[0]
2800
self.assertEqual(original[version], data)
2802
def make_knit_with_two_versions(self):
2803
knit = self.make_test_knit()
2804
knit.add_lines('a', [], ["foo"])
2805
knit.add_lines('b', [], ["bar"])
2808
def test_get_raw_records(self):
2809
knit = self.make_knit_with_two_versions()
2810
self.assertGetRawRecords(knit, ['a', 'b'])
2811
self.assertGetRawRecords(knit, ['a'])
2812
self.assertGetRawRecords(knit, ['b'])
2814
def test_get_raw_record_from_backing_knit(self):
2815
# the thunk layer should create an artificial A on-demand when needed.
2816
source_knit = self.make_test_knit(name='plain', annotate=False)
2817
target_knit = self.make_test_knit(name='annotated', annotate=True)
2818
source_knit.add_lines("A", [], ["Foo\n"])
2819
# Give the target A, so we can try to thunk across to it.
2820
target_knit.join(source_knit)
2821
index, access = self.get_index_access(target_knit,
2822
source_knit.get_data_stream([]))
2823
raw_data = list(access.get_raw_records([(True, "A", None, None)]))[0]
2824
df = GzipFile(mode='rb', fileobj=StringIO(raw_data))
2826
'version A 1 5d36b88bb697a2d778f024048bafabd443d74503\n'
2830
def test_asking_for_thunk_stream_is_not_plain_errors(self):
2831
knit = self.make_test_knit(name='annotated', annotate=True)
2832
knit.add_lines("A", [], ["Foo\n"])
2833
index, access = self.get_index_access(knit,
2834
knit.get_data_stream([]))
2835
self.assertRaises(errors.KnitCorrupt,
2836
list, access.get_raw_records([(True, "A", None, None)]))
2839
class TestFormatSignatures(KnitTests):
2841
def test_knit_format_signatures(self):
2842
"""Different formats of knit have different signature strings."""
2843
knit = self.make_test_knit(name='a', annotate=True)
2844
self.assertEqual('knit-annotated', knit.get_format_signature())
2845
knit = self.make_test_knit(name='p', annotate=False)
2846
self.assertEqual('knit-plain', knit.get_format_signature())
added
removed
bzrlib/tests/test_knit.py
