/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Martin Pool
  • Date: 2008-11-27 07:25:52 UTC
  • mfrom: (3789.2.15 pack_retry_153786)
  • mto: This revision was merged to the branch mainline in revision 3865.
  • Revision ID: mbp@sourcefrog.net-20081127072552-st8jnmahi0iy3lrt
merge fix to retry when packs have change

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
48
48
    _KnitKeyAccess,
49
49
    make_file_factory,
50
50
    )
 
51
from bzrlib.repofmt import pack_repo
51
52
from bzrlib.tests import (
52
53
    Feature,
53
54
    KnownFailure,
270
271
        return queue_call
271
272
 
272
273
 
 
274
class MockReadvFailingTransport(MockTransport):
 
275
    """Fail in the middle of a readv() result.
 
276
 
 
277
    This Transport will successfully yield the first two requested hunks, but
 
278
    raise NoSuchFile for the rest.
 
279
    """
 
280
 
 
281
    def readv(self, relpath, offsets):
 
282
        count = 0
 
283
        for result in MockTransport.readv(self, relpath, offsets):
 
284
            count += 1
 
285
            # we use 2 because the first offset is the pack header, the second
 
286
            # is the first actual content requset
 
287
            if count > 2:
 
288
                raise errors.NoSuchFile(relpath)
 
289
            yield result
 
290
 
 
291
 
273
292
class KnitRecordAccessTestsMixin(object):
274
293
    """Tests for getting and putting knit records."""
275
294
 
304
323
        mapper = ConstantMapper("foo")
305
324
        access = _KnitKeyAccess(self.get_transport(), mapper)
306
325
        return access
307
 
    
 
326
 
 
327
 
 
328
class _TestException(Exception):
 
329
    """Just an exception for local tests to use."""
 
330
 
308
331
 
309
332
class TestPackKnitAccess(TestCaseWithMemoryTransport, KnitRecordAccessTestsMixin):
310
333
    """Tests for the pack based access."""
322
345
        access.set_writer(writer, index, (transport, packname))
323
346
        return access, writer
324
347
 
 
348
    def make_pack_file(self):
 
349
        """Create a pack file with 2 records."""
 
350
        access, writer = self._get_access(packname='packname', index='foo')
 
351
        memos = []
 
352
        memos.extend(access.add_raw_records([('key1', 10)], '1234567890'))
 
353
        memos.extend(access.add_raw_records([('key2', 5)], '12345'))
 
354
        writer.end()
 
355
        return memos
 
356
 
 
357
    def make_vf_for_retrying(self):
 
358
        """Create 3 packs and a reload function.
 
359
 
 
360
        Originally, 2 pack files will have the data, but one will be missing.
 
361
        And then the third will be used in place of the first two if reload()
 
362
        is called.
 
363
 
 
364
        :return: (versioned_file, reload_counter)
 
365
            versioned_file  a KnitVersionedFiles using the packs for access
 
366
        """
 
367
        tree = self.make_branch_and_memory_tree('tree')
 
368
        tree.lock_write()
 
369
        try:
 
370
            tree.add([''], ['root-id'])
 
371
            tree.commit('one', rev_id='rev-1')
 
372
            tree.commit('two', rev_id='rev-2')
 
373
            tree.commit('three', rev_id='rev-3')
 
374
            # Pack these two revisions into another pack file, but don't remove
 
375
            # the originials
 
376
            repo = tree.branch.repository
 
377
            collection = repo._pack_collection
 
378
            collection.ensure_loaded()
 
379
            orig_packs = collection.packs
 
380
            packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
381
            new_pack = packer.pack()
 
382
 
 
383
            vf = tree.branch.repository.revisions
 
384
        finally:
 
385
            tree.unlock()
 
386
        tree.branch.repository.lock_read()
 
387
        self.addCleanup(tree.branch.repository.unlock)
 
388
        del tree
 
389
        # Set up a reload() function that switches to using the new pack file
 
390
        new_index = new_pack.revision_index
 
391
        access_tuple = new_pack.access_tuple()
 
392
        reload_counter = [0, 0, 0]
 
393
        def reload():
 
394
            reload_counter[0] += 1
 
395
            if reload_counter[1] > 0:
 
396
                # We already reloaded, nothing more to do
 
397
                reload_counter[2] += 1
 
398
                return False
 
399
            reload_counter[1] += 1
 
400
            vf._index._graph_index._indices[:] = [new_index]
 
401
            vf._access._indices.clear()
 
402
            vf._access._indices[new_index] = access_tuple
 
403
            return True
 
404
        # Delete one of the pack files so the data will need to be reloaded. We
 
405
        # will delete the file with 'rev-2' in it
 
406
        trans, name = orig_packs[1].access_tuple()
 
407
        trans.delete(name)
 
408
        # We don't have the index trigger reloading because we want to test
 
409
        # that we reload when the .pack disappears
 
410
        vf._access._reload_func = reload
 
411
        return vf, reload_counter
 
412
 
 
413
    def make_reload_func(self, return_val=True):
 
414
        reload_called = [0]
 
415
        def reload():
 
416
            reload_called[0] += 1
 
417
            return return_val
 
418
        return reload_called, reload
 
419
 
 
420
    def make_retry_exception(self):
 
421
        # We raise a real exception so that sys.exc_info() is properly
 
422
        # populated
 
423
        try:
 
424
            raise _TestException('foobar')
 
425
        except _TestException, e:
 
426
            retry_exc = errors.RetryWithNewPacks(reload_occurred=False,
 
427
                                                 exc_info=sys.exc_info())
 
428
        return retry_exc
 
429
 
325
430
    def test_read_from_several_packs(self):
326
431
        access, writer = self._get_access()
327
432
        memos = []
363
468
        writer.end()
364
469
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
365
470
 
 
471
    def test_missing_index_raises_retry(self):
 
472
        memos = self.make_pack_file()
 
473
        transport = self.get_transport()
 
474
        reload_called, reload_func = self.make_reload_func()
 
475
        # Note that the index key has changed from 'foo' to 'bar'
 
476
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
477
                                   reload_func=reload_func)
 
478
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
479
                                  access.get_raw_records, memos)
 
480
        # Because a key was passed in which does not match our index list, we
 
481
        # assume that the listing was already reloaded
 
482
        self.assertTrue(e.reload_occurred)
 
483
        self.assertIsInstance(e.exc_info, tuple)
 
484
        self.assertIs(e.exc_info[0], KeyError)
 
485
        self.assertIsInstance(e.exc_info[1], KeyError)
 
486
 
 
487
    def test_missing_index_raises_key_error_with_no_reload(self):
 
488
        memos = self.make_pack_file()
 
489
        transport = self.get_transport()
 
490
        # Note that the index key has changed from 'foo' to 'bar'
 
491
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
492
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
 
493
 
 
494
    def test_missing_file_raises_retry(self):
 
495
        memos = self.make_pack_file()
 
496
        transport = self.get_transport()
 
497
        reload_called, reload_func = self.make_reload_func()
 
498
        # Note that the 'filename' has been changed to 'different-packname'
 
499
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
500
                                   reload_func=reload_func)
 
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
502
                                  access.get_raw_records, memos)
 
503
        # The file has gone missing, so we assume we need to reload
 
504
        self.assertFalse(e.reload_occurred)
 
505
        self.assertIsInstance(e.exc_info, tuple)
 
506
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
507
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
508
        self.assertEqual('different-packname', e.exc_info[1].path)
 
509
 
 
510
    def test_missing_file_raises_no_such_file_with_no_reload(self):
 
511
        memos = self.make_pack_file()
 
512
        transport = self.get_transport()
 
513
        # Note that the 'filename' has been changed to 'different-packname'
 
514
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
515
        e = self.assertListRaises(errors.NoSuchFile,
 
516
                                  access.get_raw_records, memos)
 
517
 
 
518
    def test_failing_readv_raises_retry(self):
 
519
        memos = self.make_pack_file()
 
520
        transport = self.get_transport()
 
521
        failing_transport = MockReadvFailingTransport(
 
522
                                [transport.get_bytes('packname')])
 
523
        reload_called, reload_func = self.make_reload_func()
 
524
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
525
                                   reload_func=reload_func)
 
526
        # Asking for a single record will not trigger the Mock failure
 
527
        self.assertEqual(['1234567890'],
 
528
            list(access.get_raw_records(memos[:1])))
 
529
        self.assertEqual(['12345'],
 
530
            list(access.get_raw_records(memos[1:2])))
 
531
        # A multiple offset readv() will fail mid-way through
 
532
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
533
                                  access.get_raw_records, memos)
 
534
        # The file has gone missing, so we assume we need to reload
 
535
        self.assertFalse(e.reload_occurred)
 
536
        self.assertIsInstance(e.exc_info, tuple)
 
537
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
538
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
539
        self.assertEqual('packname', e.exc_info[1].path)
 
540
 
 
541
    def test_failing_readv_raises_no_such_file_with_no_reload(self):
 
542
        memos = self.make_pack_file()
 
543
        transport = self.get_transport()
 
544
        failing_transport = MockReadvFailingTransport(
 
545
                                [transport.get_bytes('packname')])
 
546
        reload_called, reload_func = self.make_reload_func()
 
547
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
548
        # Asking for a single record will not trigger the Mock failure
 
549
        self.assertEqual(['1234567890'],
 
550
            list(access.get_raw_records(memos[:1])))
 
551
        self.assertEqual(['12345'],
 
552
            list(access.get_raw_records(memos[1:2])))
 
553
        # A multiple offset readv() will fail mid-way through
 
554
        e = self.assertListRaises(errors.NoSuchFile,
 
555
                                  access.get_raw_records, memos)
 
556
 
 
557
    def test_reload_or_raise_no_reload(self):
 
558
        access = _DirectPackAccess({}, reload_func=None)
 
559
        retry_exc = self.make_retry_exception()
 
560
        # Without a reload_func, we will just re-raise the original exception
 
561
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
562
 
 
563
    def test_reload_or_raise_reload_changed(self):
 
564
        reload_called, reload_func = self.make_reload_func(return_val=True)
 
565
        access = _DirectPackAccess({}, reload_func=reload_func)
 
566
        retry_exc = self.make_retry_exception()
 
567
        access.reload_or_raise(retry_exc)
 
568
        self.assertEqual([1], reload_called)
 
569
        retry_exc.reload_occurred=True
 
570
        access.reload_or_raise(retry_exc)
 
571
        self.assertEqual([2], reload_called)
 
572
 
 
573
    def test_reload_or_raise_reload_no_change(self):
 
574
        reload_called, reload_func = self.make_reload_func(return_val=False)
 
575
        access = _DirectPackAccess({}, reload_func=reload_func)
 
576
        retry_exc = self.make_retry_exception()
 
577
        # If reload_occurred is False, then we consider it an error to have
 
578
        # reload_func() return False (no changes).
 
579
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
580
        self.assertEqual([1], reload_called)
 
581
        retry_exc.reload_occurred=True
 
582
        # If reload_occurred is True, then we assume nothing changed because
 
583
        # it had changed earlier, but didn't change again
 
584
        access.reload_or_raise(retry_exc)
 
585
        self.assertEqual([2], reload_called)
 
586
 
 
587
    def test_annotate_retries(self):
 
588
        vf, reload_counter = self.make_vf_for_retrying()
 
589
        # It is a little bit bogus to annotate the Revision VF, but it works,
 
590
        # as we have ancestry stored there
 
591
        key = ('rev-3',)
 
592
        reload_lines = vf.annotate(key)
 
593
        self.assertEqual([1, 1, 0], reload_counter)
 
594
        plain_lines = vf.annotate(key)
 
595
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
596
        if reload_lines != plain_lines:
 
597
            self.fail('Annotation was not identical with reloading.')
 
598
        # Now delete the packs-in-use, which should trigger another reload, but
 
599
        # this time we just raise an exception because we can't recover
 
600
        for trans, name in vf._access._indices.itervalues():
 
601
            trans.delete(name)
 
602
        self.assertRaises(errors.NoSuchFile, vf.annotate, key)
 
603
        self.assertEqual([2, 1, 1], reload_counter)
 
604
 
 
605
    def test__get_record_map_retries(self):
 
606
        vf, reload_counter = self.make_vf_for_retrying()
 
607
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
608
        records = vf._get_record_map(keys)
 
609
        self.assertEqual(keys, sorted(records.keys()))
 
610
        self.assertEqual([1, 1, 0], reload_counter)
 
611
        # Now delete the packs-in-use, which should trigger another reload, but
 
612
        # this time we just raise an exception because we can't recover
 
613
        for trans, name in vf._access._indices.itervalues():
 
614
            trans.delete(name)
 
615
        self.assertRaises(errors.NoSuchFile, vf._get_record_map, keys)
 
616
        self.assertEqual([2, 1, 1], reload_counter)
 
617
 
 
618
    def test_get_record_stream_retries(self):
 
619
        vf, reload_counter = self.make_vf_for_retrying()
 
620
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
621
        record_stream = vf.get_record_stream(keys, 'topological', False)
 
622
        record = record_stream.next()
 
623
        self.assertEqual(('rev-1',), record.key)
 
624
        self.assertEqual([0, 0, 0], reload_counter)
 
625
        record = record_stream.next()
 
626
        self.assertEqual(('rev-2',), record.key)
 
627
        self.assertEqual([1, 1, 0], reload_counter)
 
628
        record = record_stream.next()
 
629
        self.assertEqual(('rev-3',), record.key)
 
630
        self.assertEqual([1, 1, 0], reload_counter)
 
631
        # Now delete all pack files, and see that we raise the right error
 
632
        for trans, name in vf._access._indices.itervalues():
 
633
            trans.delete(name)
 
634
        self.assertListRaises(errors.NoSuchFile,
 
635
            vf.get_record_stream, keys, 'topological', False)
 
636
 
 
637
    def test_iter_lines_added_or_present_in_keys_retries(self):
 
638
        vf, reload_counter = self.make_vf_for_retrying()
 
639
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
640
        # Unfortunately, iter_lines_added_or_present_in_keys iterates the
 
641
        # result in random order (determined by the iteration order from a
 
642
        # set()), so we don't have any solid way to trigger whether data is
 
643
        # read before or after. However we tried to delete the middle node to
 
644
        # exercise the code well.
 
645
        # What we care about is that all lines are always yielded, but not
 
646
        # duplicated
 
647
        count = 0
 
648
        reload_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
649
        self.assertEqual([1, 1, 0], reload_counter)
 
650
        # Now do it again, to make sure the result is equivalent
 
651
        plain_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
652
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
653
        self.assertEqual(plain_lines, reload_lines)
 
654
        self.assertEqual(21, len(plain_lines))
 
655
        # Now delete all pack files, and see that we raise the right error
 
656
        for trans, name in vf._access._indices.itervalues():
 
657
            trans.delete(name)
 
658
        self.assertListRaises(errors.NoSuchFile,
 
659
            vf.iter_lines_added_or_present_in_keys, keys)
 
660
        self.assertEqual([2, 1, 1], reload_counter)
 
661
 
366
662
 
367
663
class LowLevelKnitDataTests(TestCase):
368
664
 
373
669
        gz_file.close()
374
670
        return sio.getvalue()
375
671
 
 
672
    def make_multiple_records(self):
 
673
        """Create the content for multiple records."""
 
674
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
675
        total_txt = []
 
676
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
 
677
                                        'foo\n'
 
678
                                        'bar\n'
 
679
                                        'end rev-id-1\n'
 
680
                                        % (sha1sum,))
 
681
        record_1 = (0, len(gz_txt), sha1sum)
 
682
        total_txt.append(gz_txt)
 
683
        sha1sum = osutils.sha('baz\n').hexdigest()
 
684
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
 
685
                                        'baz\n'
 
686
                                        'end rev-id-2\n'
 
687
                                        % (sha1sum,))
 
688
        record_2 = (record_1[1], len(gz_txt), sha1sum)
 
689
        total_txt.append(gz_txt)
 
690
        return total_txt, record_1, record_2
 
691
 
376
692
    def test_valid_knit_data(self):
377
693
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
378
694
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
392
708
        raw_contents = list(knit._read_records_iter_raw(records))
393
709
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
394
710
 
 
711
    def test_multiple_records_valid(self):
 
712
        total_txt, record_1, record_2 = self.make_multiple_records()
 
713
        transport = MockTransport([''.join(total_txt)])
 
714
        access = _KnitKeyAccess(transport, ConstantMapper('filename'))
 
715
        knit = KnitVersionedFiles(None, access)
 
716
        records = [(('rev-id-1',), (('rev-id-1',), record_1[0], record_1[1])),
 
717
                   (('rev-id-2',), (('rev-id-2',), record_2[0], record_2[1]))]
 
718
 
 
719
        contents = list(knit._read_records_iter(records))
 
720
        self.assertEqual([(('rev-id-1',), ['foo\n', 'bar\n'], record_1[2]),
 
721
                          (('rev-id-2',), ['baz\n'], record_2[2])],
 
722
                         contents)
 
723
 
 
724
        raw_contents = list(knit._read_records_iter_raw(records))
 
725
        self.assertEqual([(('rev-id-1',), total_txt[0], record_1[2]),
 
726
                          (('rev-id-2',), total_txt[1], record_2[2])],
 
727
                         raw_contents)
 
728
 
395
729
    def test_not_enough_lines(self):
396
730
        sha1sum = osutils.sha('foo\n').hexdigest()
397
731
        # record says 2 lines data says 1