/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
1
# Copyright (C) 2008 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Tests for pack repositories.
18
19
These tests are repeated for all pack-based repository formats.
20
"""
21
3582.3.4 by Martin Pool
Use cStringIO rather than StringIO
22
from cStringIO import StringIO
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
23
from stat import S_ISDIR
24
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
25
from bzrlib.btree_index import BTreeGraphIndex
26
from bzrlib.index import GraphIndex
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
27
from bzrlib import (
28
    bzrdir,
29
    errors,
30
    inventory,
4002.1.1 by Andrew Bennetts
Implement suspend_write_group/resume_write_group.
31
    osutils,
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
32
    progress,
33
    repository,
34
    revision as _mod_revision,
35
    symbol_versioning,
36
    tests,
37
    ui,
38
    upgrade,
39
    workingtree,
40
    )
3801.1.18 by Andrew Bennetts
Add a test that ensures that the autopack RPC is actually used for all pack formats.
41
from bzrlib.smart import (
42
    client,
43
    server,
44
    )
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
45
from bzrlib.tests import (
46
    TestCase,
47
    TestCaseWithTransport,
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
48
    TestNotApplicable,
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
49
    TestSkipped,
50
    )
51
from bzrlib.transport import (
52
    fakenfs,
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
53
    memory,
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
54
    get_transport,
55
    )
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
56
from bzrlib.tests.per_repository import TestCaseWithRepository
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
57
58
59
class TestPackRepository(TestCaseWithTransport):
60
    """Tests to be repeated across all pack-based formats.
61
62
    The following are populated from the test scenario:
63
64
    :ivar format_name: Registered name fo the format to test.
65
    :ivar format_string: On-disk format marker.
66
    :ivar format_supports_external_lookups: Boolean.
67
    """
68
69
    def get_format(self):
70
        return bzrdir.format_registry.make_bzrdir(self.format_name)
71
72
    def test_attribute__fetch_order(self):
3606.7.3 by John Arbash Meinel
We don't have to fetch in topological order, as long as we fix all of the delta logic pieces.
73
        """Packs do not need ordered data retrieval."""
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
74
        format = self.get_format()
75
        repo = self.make_repository('.', format=format)
3606.7.8 by John Arbash Meinel
Switch names to 'unordered' that I missed before.
76
        self.assertEqual('unordered', repo._fetch_order)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
77
78
    def test_attribute__fetch_uses_deltas(self):
79
        """Packs reuse deltas."""
80
        format = self.get_format()
81
        repo = self.make_repository('.', format=format)
82
        self.assertEqual(True, repo._fetch_uses_deltas)
83
84
    def test_disk_layout(self):
85
        format = self.get_format()
86
        repo = self.make_repository('.', format=format)
87
        # in case of side effects of locking.
88
        repo.lock_write()
89
        repo.unlock()
90
        t = repo.bzrdir.get_repository_transport(None)
91
        self.check_format(t)
92
        # XXX: no locks left when unlocked at the moment
93
        # self.assertEqualDiff('', t.get('lock').read())
94
        self.check_databases(t)
95
96
    def check_format(self, t):
97
        self.assertEqualDiff(
98
            self.format_string, # from scenario
99
            t.get('format').read())
100
101
    def assertHasNoKndx(self, t, knit_name):
102
        """Assert that knit_name has no index on t."""
103
        self.assertFalse(t.has(knit_name + '.kndx'))
104
105
    def assertHasNoKnit(self, t, knit_name):
106
        """Assert that knit_name exists on t."""
107
        # no default content
108
        self.assertFalse(t.has(knit_name + '.knit'))
109
110
    def check_databases(self, t):
111
        """check knit content for a repository."""
112
        # check conversion worked
113
        self.assertHasNoKndx(t, 'inventory')
114
        self.assertHasNoKnit(t, 'inventory')
115
        self.assertHasNoKndx(t, 'revisions')
116
        self.assertHasNoKnit(t, 'revisions')
117
        self.assertHasNoKndx(t, 'signatures')
118
        self.assertHasNoKnit(t, 'signatures')
119
        self.assertFalse(t.has('knits'))
120
        # revision-indexes file-container directory
121
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
122
            list(self.index_class(t, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
123
        self.assertTrue(S_ISDIR(t.stat('packs').st_mode))
124
        self.assertTrue(S_ISDIR(t.stat('upload').st_mode))
125
        self.assertTrue(S_ISDIR(t.stat('indices').st_mode))
126
        self.assertTrue(S_ISDIR(t.stat('obsolete_packs').st_mode))
127
128
    def test_shared_disk_layout(self):
129
        format = self.get_format()
130
        repo = self.make_repository('.', shared=True, format=format)
131
        # we want:
132
        t = repo.bzrdir.get_repository_transport(None)
133
        self.check_format(t)
134
        # XXX: no locks left when unlocked at the moment
135
        # self.assertEqualDiff('', t.get('lock').read())
136
        # We should have a 'shared-storage' marker file.
137
        self.assertEqualDiff('', t.get('shared-storage').read())
138
        self.check_databases(t)
139
140
    def test_shared_no_tree_disk_layout(self):
141
        format = self.get_format()
142
        repo = self.make_repository('.', shared=True, format=format)
143
        repo.set_make_working_trees(False)
144
        # we want:
145
        t = repo.bzrdir.get_repository_transport(None)
146
        self.check_format(t)
147
        # XXX: no locks left when unlocked at the moment
148
        # self.assertEqualDiff('', t.get('lock').read())
149
        # We should have a 'shared-storage' marker file.
150
        self.assertEqualDiff('', t.get('shared-storage').read())
151
        # We should have a marker for the no-working-trees flag.
152
        self.assertEqualDiff('', t.get('no-working-trees').read())
153
        # The marker should go when we toggle the setting.
154
        repo.set_make_working_trees(True)
155
        self.assertFalse(t.has('no-working-trees'))
156
        self.check_databases(t)
157
158
    def test_adding_revision_creates_pack_indices(self):
159
        format = self.get_format()
160
        tree = self.make_branch_and_tree('.', format=format)
161
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
162
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
163
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
164
        tree.commit('foobarbaz')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
165
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
166
        index_nodes = list(index.iter_all_entries())
167
        self.assertEqual(1, len(index_nodes))
168
        node = index_nodes[0]
169
        name = node[1][0]
170
        # the pack sizes should be listed in the index
171
        pack_value = node[2]
172
        sizes = [int(digits) for digits in pack_value.split(' ')]
173
        for size, suffix in zip(sizes, ['.rix', '.iix', '.tix', '.six']):
174
            stat = trans.stat('indices/%s%s' % (name, suffix))
175
            self.assertEqual(size, stat.st_size)
176
177
    def test_pulling_nothing_leads_to_no_new_names(self):
178
        format = self.get_format()
179
        tree1 = self.make_branch_and_tree('1', format=format)
180
        tree2 = self.make_branch_and_tree('2', format=format)
181
        tree1.branch.repository.fetch(tree2.branch.repository)
182
        trans = tree1.branch.repository.bzrdir.get_repository_transport(None)
183
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
184
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
185
186
    def test_commit_across_pack_shape_boundary_autopacks(self):
187
        format = self.get_format()
188
        tree = self.make_branch_and_tree('.', format=format)
189
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
190
        # This test could be a little cheaper by replacing the packs
191
        # attribute on the repository to allow a different pack distribution
192
        # and max packs policy - so we are checking the policy is honoured
193
        # in the test. But for now 11 commits is not a big deal in a single
194
        # test.
195
        for x in range(9):
196
            tree.commit('commit %s' % x)
197
        # there should be 9 packs:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
198
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
199
        self.assertEqual(9, len(list(index.iter_all_entries())))
200
        # insert some files in obsolete_packs which should be removed by pack.
201
        trans.put_bytes('obsolete_packs/foo', '123')
202
        trans.put_bytes('obsolete_packs/bar', '321')
203
        # committing one more should coalesce to 1 of 10.
204
        tree.commit('commit triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
205
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
206
        self.assertEqual(1, len(list(index.iter_all_entries())))
207
        # packing should not damage data
208
        tree = tree.bzrdir.open_workingtree()
209
        check_result = tree.branch.repository.check(
210
            [tree.branch.last_revision()])
211
        # We should have 50 (10x5) files in the obsolete_packs directory.
212
        obsolete_files = list(trans.list_dir('obsolete_packs'))
213
        self.assertFalse('foo' in obsolete_files)
214
        self.assertFalse('bar' in obsolete_files)
215
        self.assertEqual(50, len(obsolete_files))
216
        # XXX: Todo check packs obsoleted correctly - old packs and indices
217
        # in the obsolete_packs directory.
218
        large_pack_name = list(index.iter_all_entries())[0][1][0]
219
        # finally, committing again should not touch the large pack.
220
        tree.commit('commit not triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
221
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
222
        self.assertEqual(2, len(list(index.iter_all_entries())))
223
        pack_names = [node[1][0] for node in index.iter_all_entries()]
224
        self.assertTrue(large_pack_name in pack_names)
225
226
    def test_fail_obsolete_deletion(self):
227
        # failing to delete obsolete packs is not fatal
228
        format = self.get_format()
229
        server = fakenfs.FakeNFSServer()
230
        server.setUp()
231
        self.addCleanup(server.tearDown)
232
        transport = get_transport(server.get_url())
233
        bzrdir = self.get_format().initialize_on_transport(transport)
234
        repo = bzrdir.create_repository()
235
        repo_transport = bzrdir.get_repository_transport(None)
236
        self.assertTrue(repo_transport.has('obsolete_packs'))
237
        # these files are in use by another client and typically can't be deleted
238
        repo_transport.put_bytes('obsolete_packs/.nfsblahblah', 'contents')
239
        repo._pack_collection._clear_obsolete_packs()
240
        self.assertTrue(repo_transport.has('obsolete_packs/.nfsblahblah'))
241
242
    def test_pack_after_two_commits_packs_everything(self):
243
        format = self.get_format()
244
        tree = self.make_branch_and_tree('.', format=format)
245
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
246
        tree.commit('start')
247
        tree.commit('more work')
248
        tree.branch.repository.pack()
249
        # there should be 1 pack:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
250
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
251
        self.assertEqual(1, len(list(index.iter_all_entries())))
252
        self.assertEqual(2, len(tree.branch.repository.all_revision_ids()))
253
254
    def test_pack_layout(self):
255
        format = self.get_format()
256
        tree = self.make_branch_and_tree('.', format=format)
257
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
258
        tree.commit('start', rev_id='1')
259
        tree.commit('more work', rev_id='2')
260
        tree.branch.repository.pack()
261
        tree.lock_read()
262
        self.addCleanup(tree.unlock)
263
        pack = tree.branch.repository._pack_collection.get_pack_by_name(
264
            tree.branch.repository._pack_collection.names()[0])
265
        # revision access tends to be tip->ancestor, so ordering that way on 
266
        # disk is a good idea.
267
        for _1, key, val, refs in pack.revision_index.iter_all_entries():
268
            if key == ('1',):
269
                pos_1 = int(val[1:].split()[0])
270
            else:
271
                pos_2 = int(val[1:].split()[0])
272
        self.assertTrue(pos_2 < pos_1)
273
274
    def test_pack_repositories_support_multiple_write_locks(self):
275
        format = self.get_format()
276
        self.make_repository('.', shared=True, format=format)
277
        r1 = repository.Repository.open('.')
278
        r2 = repository.Repository.open('.')
279
        r1.lock_write()
280
        self.addCleanup(r1.unlock)
281
        r2.lock_write()
282
        r2.unlock()
283
284
    def _add_text(self, repo, fileid):
285
        """Add a text to the repository within a write group."""
286
        repo.texts.add_lines((fileid, 'samplerev+'+fileid), [], [])
287
288
    def test_concurrent_writers_merge_new_packs(self):
289
        format = self.get_format()
290
        self.make_repository('.', shared=True, format=format)
291
        r1 = repository.Repository.open('.')
292
        r2 = repository.Repository.open('.')
293
        r1.lock_write()
294
        try:
295
            # access enough data to load the names list
296
            list(r1.all_revision_ids())
297
            r2.lock_write()
298
            try:
299
                # access enough data to load the names list
300
                list(r2.all_revision_ids())
301
                r1.start_write_group()
302
                try:
303
                    r2.start_write_group()
304
                    try:
305
                        self._add_text(r1, 'fileidr1')
306
                        self._add_text(r2, 'fileidr2')
307
                    except:
308
                        r2.abort_write_group()
309
                        raise
310
                except:
311
                    r1.abort_write_group()
312
                    raise
313
                # both r1 and r2 have open write groups with data in them
314
                # created while the other's write group was open.
315
                # Commit both which requires a merge to the pack-names.
316
                try:
317
                    r1.commit_write_group()
318
                except:
319
                    r1.abort_write_group()
320
                    r2.abort_write_group()
321
                    raise
322
                r2.commit_write_group()
323
                # tell r1 to reload from disk
324
                r1._pack_collection.reset()
325
                # Now both repositories should know about both names
326
                r1._pack_collection.ensure_loaded()
327
                r2._pack_collection.ensure_loaded()
328
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
329
                self.assertEqual(2, len(r1._pack_collection.names()))
330
            finally:
331
                r2.unlock()
332
        finally:
333
            r1.unlock()
334
335
    def test_concurrent_writer_second_preserves_dropping_a_pack(self):
336
        format = self.get_format()
337
        self.make_repository('.', shared=True, format=format)
338
        r1 = repository.Repository.open('.')
339
        r2 = repository.Repository.open('.')
340
        # add a pack to drop
341
        r1.lock_write()
342
        try:
343
            r1.start_write_group()
344
            try:
345
                self._add_text(r1, 'fileidr1')
346
            except:
347
                r1.abort_write_group()
348
                raise
349
            else:
350
                r1.commit_write_group()
351
            r1._pack_collection.ensure_loaded()
352
            name_to_drop = r1._pack_collection.all_packs()[0].name
353
        finally:
354
            r1.unlock()
355
        r1.lock_write()
356
        try:
357
            # access enough data to load the names list
358
            list(r1.all_revision_ids())
359
            r2.lock_write()
360
            try:
361
                # access enough data to load the names list
362
                list(r2.all_revision_ids())
363
                r1._pack_collection.ensure_loaded()
364
                try:
365
                    r2.start_write_group()
366
                    try:
367
                        # in r1, drop the pack
368
                        r1._pack_collection._remove_pack_from_memory(
369
                            r1._pack_collection.get_pack_by_name(name_to_drop))
370
                        # in r2, add a pack
371
                        self._add_text(r2, 'fileidr2')
372
                    except:
373
                        r2.abort_write_group()
374
                        raise
375
                except:
376
                    r1._pack_collection.reset()
377
                    raise
378
                # r1 has a changed names list, and r2 an open write groups with
379
                # changes.
380
                # save r1, and then commit the r2 write group, which requires a
381
                # merge to the pack-names, which should not reinstate
382
                # name_to_drop
383
                try:
384
                    r1._pack_collection._save_pack_names()
385
                    r1._pack_collection.reset()
386
                except:
387
                    r2.abort_write_group()
388
                    raise
389
                try:
390
                    r2.commit_write_group()
391
                except:
392
                    r2.abort_write_group()
393
                    raise
394
                # Now both repositories should now about just one name.
395
                r1._pack_collection.ensure_loaded()
396
                r2._pack_collection.ensure_loaded()
397
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
398
                self.assertEqual(1, len(r1._pack_collection.names()))
399
                self.assertFalse(name_to_drop in r1._pack_collection.names())
400
            finally:
401
                r2.unlock()
402
        finally:
403
            r1.unlock()
404
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
405
    def test_concurrent_pack_triggers_reload(self):
406
        # create 2 packs, which we will then collapse
407
        tree = self.make_branch_and_tree('tree')
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
408
        tree.lock_write()
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
409
        try:
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
410
            rev1 = tree.commit('one')
411
            rev2 = tree.commit('two')
412
            r2 = repository.Repository.open('tree')
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
413
            r2.lock_read()
414
            try:
415
                # Now r2 has read the pack-names file, but will need to reload
416
                # it after r1 has repacked
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
417
                tree.branch.repository.pack()
418
                self.assertEqual({rev2:(rev1,)}, r2.get_parent_map([rev2]))
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
419
            finally:
420
                r2.unlock()
421
        finally:
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
422
            tree.unlock()
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
423
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
424
    def test_concurrent_pack_during_get_record_reloads(self):
425
        tree = self.make_branch_and_tree('tree')
426
        tree.lock_write()
427
        try:
428
            rev1 = tree.commit('one')
429
            rev2 = tree.commit('two')
3789.2.14 by John Arbash Meinel
Update AggregateIndex to pass the reload_func into _DirectPackAccess
430
            keys = [(rev1,), (rev2,)]
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
431
            r2 = repository.Repository.open('tree')
432
            r2.lock_read()
433
            try:
434
                # At this point, we will start grabbing a record stream, and
435
                # trigger a repack mid-way
436
                packed = False
437
                result = {}
438
                record_stream = r2.revisions.get_record_stream(keys,
439
                                    'unordered', False)
440
                for record in record_stream:
441
                    result[record.key] = record
442
                    if not packed:
443
                        tree.branch.repository.pack()
444
                        packed = True
445
                # The first record will be found in the original location, but
446
                # after the pack, we have to reload to find the next record
3789.2.14 by John Arbash Meinel
Update AggregateIndex to pass the reload_func into _DirectPackAccess
447
                self.assertEqual(sorted(keys), sorted(result.keys()))
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
448
            finally:
449
                r2.unlock()
450
        finally:
451
            tree.unlock()
452
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
453
    def test_lock_write_does_not_physically_lock(self):
454
        repo = self.make_repository('.', format=self.get_format())
455
        repo.lock_write()
456
        self.addCleanup(repo.unlock)
457
        self.assertFalse(repo.get_physical_lock_status())
458
459
    def prepare_for_break_lock(self):
460
        # Setup the global ui factory state so that a break-lock method call
461
        # will find usable input in the input stream.
462
        old_factory = ui.ui_factory
463
        def restoreFactory():
464
            ui.ui_factory = old_factory
465
        self.addCleanup(restoreFactory)
466
        ui.ui_factory = ui.SilentUIFactory()
467
        ui.ui_factory.stdin = StringIO("y\n")
468
469
    def test_break_lock_breaks_physical_lock(self):
470
        repo = self.make_repository('.', format=self.get_format())
471
        repo._pack_collection.lock_names()
3650.4.1 by Aaron Bentley
Fix test kipple in test_break_lock_breaks_physical_lock
472
        repo.control_files.leave_in_place()
473
        repo.unlock()
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
474
        repo2 = repository.Repository.open('.')
475
        self.assertTrue(repo.get_physical_lock_status())
476
        self.prepare_for_break_lock()
477
        repo2.break_lock()
478
        self.assertFalse(repo.get_physical_lock_status())
479
480
    def test_broken_physical_locks_error_on__unlock_names_lock(self):
481
        repo = self.make_repository('.', format=self.get_format())
482
        repo._pack_collection.lock_names()
483
        self.assertTrue(repo.get_physical_lock_status())
484
        repo2 = repository.Repository.open('.')
485
        self.prepare_for_break_lock()
486
        repo2.break_lock()
487
        self.assertRaises(errors.LockBroken, repo._pack_collection._unlock_names)
488
489
    def test_fetch_without_find_ghosts_ignores_ghosts(self):
490
        # we want two repositories at this point:
491
        # one with a revision that is a ghost in the other
492
        # repository.
493
        # 'ghost' is present in has_ghost, 'ghost' is absent in 'missing_ghost'.
494
        # 'references' is present in both repositories, and 'tip' is present
495
        # just in has_ghost.
496
        # has_ghost       missing_ghost
497
        #------------------------------
498
        # 'ghost'             -
499
        # 'references'    'references'
500
        # 'tip'               -
501
        # In this test we fetch 'tip' which should not fetch 'ghost'
502
        has_ghost = self.make_repository('has_ghost', format=self.get_format())
503
        missing_ghost = self.make_repository('missing_ghost',
504
            format=self.get_format())
505
506
        def add_commit(repo, revision_id, parent_ids):
507
            repo.lock_write()
508
            repo.start_write_group()
509
            inv = inventory.Inventory(revision_id=revision_id)
510
            inv.root.revision = revision_id
511
            root_id = inv.root.file_id
512
            sha1 = repo.add_inventory(revision_id, inv, [])
513
            repo.texts.add_lines((root_id, revision_id), [], [])
514
            rev = _mod_revision.Revision(timestamp=0,
515
                                         timezone=None,
516
                                         committer="Foo Bar <foo@example.com>",
517
                                         message="Message",
518
                                         inventory_sha1=sha1,
519
                                         revision_id=revision_id)
520
            rev.parent_ids = parent_ids
521
            repo.add_revision(revision_id, rev)
522
            repo.commit_write_group()
523
            repo.unlock()
524
        add_commit(has_ghost, 'ghost', [])
525
        add_commit(has_ghost, 'references', ['ghost'])
526
        add_commit(missing_ghost, 'references', ['ghost'])
527
        add_commit(has_ghost, 'tip', ['references'])
528
        missing_ghost.fetch(has_ghost, 'tip')
529
        # missing ghost now has tip and not ghost.
530
        rev = missing_ghost.get_revision('tip')
531
        inv = missing_ghost.get_inventory('tip')
532
        self.assertRaises(errors.NoSuchRevision,
533
            missing_ghost.get_revision, 'ghost')
534
        self.assertRaises(errors.NoSuchRevision,
535
            missing_ghost.get_inventory, 'ghost')
536
537
    def test_supports_external_lookups(self):
538
        repo = self.make_repository('.', format=self.get_format())
539
        self.assertEqual(self.format_supports_external_lookups,
540
            repo._format.supports_external_lookups)
541
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
542
    def test_abort_write_group_does_not_raise_when_suppressed(self):
543
        """Similar to per_repository.test_write_group's test of the same name.
544
545
        Also requires that the exception is logged.
546
        """
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
547
        self.vfs_transport_factory = memory.MemoryServer
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
548
        repo = self.make_repository('repo')
549
        token = repo.lock_write()
550
        self.addCleanup(repo.unlock)
551
        repo.start_write_group()
552
        # Damage the repository on the filesystem
553
        self.get_transport('').rename('repo', 'foo')
554
        # abort_write_group will not raise an error
555
        self.assertEqual(None, repo.abort_write_group(suppress_errors=True))
556
        # But it does log an error
557
        log_file = self._get_log(keep_log_file=True)
558
        self.assertContainsRe(log_file, 'abort_write_group failed')
559
        self.assertContainsRe(log_file, r'INFO  bzr: ERROR \(ignored\):')
560
        if token is not None:
561
            repo.leave_lock_in_place()
562
        
563
    def test_abort_write_group_does_raise_when_not_suppressed(self):
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
564
        self.vfs_transport_factory = memory.MemoryServer
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
565
        repo = self.make_repository('repo')
566
        token = repo.lock_write()
567
        self.addCleanup(repo.unlock)
568
        repo.start_write_group()
569
        # Damage the repository on the filesystem
570
        self.get_transport('').rename('repo', 'foo')
571
        # abort_write_group will not raise an error
572
        self.assertRaises(Exception, repo.abort_write_group)
573
        if token is not None:
574
            repo.leave_lock_in_place()
4002.1.1 by Andrew Bennetts
Implement suspend_write_group/resume_write_group.
575
576
    def test_suspend_write_group(self):
577
        self.vfs_transport_factory = memory.MemoryServer
578
        repo = self.make_repository('repo')
579
        token = repo.lock_write()
580
        self.addCleanup(repo.unlock)
581
        repo.start_write_group()
582
        repo.texts.add_lines(('file-id', 'revid'), (), ['lines'])
583
        wg_tokens = repo.suspend_write_group()
584
        expected_pack_name = wg_tokens[0] + '.pack'
585
        upload_transport = repo._pack_collection._upload_transport
586
        limbo_files = upload_transport.list_dir('')
587
        self.assertTrue(expected_pack_name in limbo_files, limbo_files)
588
        md5 = osutils.md5(upload_transport.get_bytes(expected_pack_name))
589
        self.assertEqual(wg_tokens[0], md5.hexdigest())
590
591
    def test_resume_write_group_then_abort(self):
592
        # Create a repo, start a write group, insert some data, suspend.
593
        self.vfs_transport_factory = memory.MemoryServer
594
        repo = self.make_repository('repo')
595
        token = repo.lock_write()
596
        self.addCleanup(repo.unlock)
597
        repo.start_write_group()
598
        text_key = ('file-id', 'revid')
599
        repo.texts.add_lines(text_key, (), ['lines'])
600
        wg_tokens = repo.suspend_write_group()
601
        # Get a fresh repository object for the repo on the filesystem.
602
        same_repo = repo.bzrdir.open_repository()
603
        # Resume
604
        same_repo.lock_write()
605
        self.addCleanup(same_repo.unlock)
606
        same_repo.resume_write_group(wg_tokens)
607
        same_repo.abort_write_group()
608
        self.assertEqual(
609
            [], same_repo._pack_collection._upload_transport.list_dir(''))
610
        self.assertEqual(
611
            [], same_repo._pack_collection._pack_transport.list_dir(''))
612
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
613
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
614
class TestPackRepositoryStacking(TestCaseWithTransport):
615
616
    """Tests for stacking pack repositories"""
617
618
    def setUp(self):
619
        if not self.format_supports_external_lookups:
620
            raise TestNotApplicable("%r doesn't support stacking" 
621
                % (self.format_name,))
622
        super(TestPackRepositoryStacking, self).setUp()
623
624
    def get_format(self):
625
        return bzrdir.format_registry.make_bzrdir(self.format_name)
626
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
627
    def test_stack_checks_rich_root_compatibility(self):
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
628
        # early versions of the packing code relied on pack internals to
629
        # stack, but the current version should be able to stack on any
630
        # format.
631
        #
632
        # TODO: Possibly this should be run per-repository-format and raise
633
        # TestNotApplicable on formats that don't support stacking. -- mbp
634
        # 20080729
635
        repo = self.make_repository('repo', format=self.get_format())
636
        if repo.supports_rich_root():
637
            # can only stack on repositories that have compatible internal
638
            # metadata
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
639
            if getattr(repo._format, 'supports_tree_reference', False):
640
                matching_format_name = 'pack-0.92-subtree'
641
            else:
642
                matching_format_name = 'rich-root-pack'
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
643
            mismatching_format_name = 'pack-0.92'
644
        else:
645
            matching_format_name = 'pack-0.92'
646
            mismatching_format_name = 'pack-0.92-subtree'
647
        base = self.make_repository('base', format=matching_format_name)
648
        repo.add_fallback_repository(base)
649
        # you can't stack on something with incompatible data
650
        bad_repo = self.make_repository('mismatch',
651
            format=mismatching_format_name)
652
        e = self.assertRaises(errors.IncompatibleRepositories,
653
            repo.add_fallback_repository, bad_repo)
654
        self.assertContainsRe(str(e),
655
            r'(?m)KnitPackRepository.*/mismatch/.*\nis not compatible with\n'
656
            r'KnitPackRepository.*/repo/.*\n'
657
            r'different rich-root support')
658
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
659
    def test_stack_checks_serializers_compatibility(self):
660
        repo = self.make_repository('repo', format=self.get_format())
661
        if getattr(repo._format, 'supports_tree_reference', False):
662
            # can only stack on repositories that have compatible internal
663
            # metadata
664
            matching_format_name = 'pack-0.92-subtree'
665
            mismatching_format_name = 'rich-root-pack'
666
        else:
667
            if repo.supports_rich_root():
668
                matching_format_name = 'rich-root-pack'
669
                mismatching_format_name = 'pack-0.92-subtree'
670
            else:
671
                raise TestNotApplicable('No formats use non-v5 serializer'
672
                    ' without having rich-root also set')
673
        base = self.make_repository('base', format=matching_format_name)
674
        repo.add_fallback_repository(base)
675
        # you can't stack on something with incompatible data
676
        bad_repo = self.make_repository('mismatch',
677
            format=mismatching_format_name)
678
        e = self.assertRaises(errors.IncompatibleRepositories,
679
            repo.add_fallback_repository, bad_repo)
680
        self.assertContainsRe(str(e),
681
            r'(?m)KnitPackRepository.*/mismatch/.*\nis not compatible with\n'
682
            r'KnitPackRepository.*/repo/.*\n'
683
            r'different serializers')
684
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
685
    def test_adding_pack_does_not_record_pack_names_from_other_repositories(self):
686
        base = self.make_branch_and_tree('base', format=self.get_format())
687
        base.commit('foo')
688
        referencing = self.make_branch_and_tree('repo', format=self.get_format())
689
        referencing.branch.repository.add_fallback_repository(base.branch.repository)
690
        referencing.commit('bar')
691
        new_instance = referencing.bzrdir.open_repository()
692
        new_instance.lock_read()
693
        self.addCleanup(new_instance.unlock)
694
        new_instance._pack_collection.ensure_loaded()
695
        self.assertEqual(1, len(new_instance._pack_collection.all_packs()))
696
697
    def test_autopack_only_considers_main_repo_packs(self):
698
        base = self.make_branch_and_tree('base', format=self.get_format())
699
        base.commit('foo')
700
        tree = self.make_branch_and_tree('repo', format=self.get_format())
701
        tree.branch.repository.add_fallback_repository(base.branch.repository)
702
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
703
        # This test could be a little cheaper by replacing the packs
704
        # attribute on the repository to allow a different pack distribution
705
        # and max packs policy - so we are checking the policy is honoured
706
        # in the test. But for now 11 commits is not a big deal in a single
707
        # test.
708
        for x in range(9):
709
            tree.commit('commit %s' % x)
710
        # there should be 9 packs:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
711
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
712
        self.assertEqual(9, len(list(index.iter_all_entries())))
713
        # committing one more should coalesce to 1 of 10.
714
        tree.commit('commit triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
715
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
716
        self.assertEqual(1, len(list(index.iter_all_entries())))
717
        # packing should not damage data
718
        tree = tree.bzrdir.open_workingtree()
719
        check_result = tree.branch.repository.check(
720
            [tree.branch.last_revision()])
721
        # We should have 50 (10x5) files in the obsolete_packs directory.
722
        obsolete_files = list(trans.list_dir('obsolete_packs'))
723
        self.assertFalse('foo' in obsolete_files)
724
        self.assertFalse('bar' in obsolete_files)
725
        self.assertEqual(50, len(obsolete_files))
726
        # XXX: Todo check packs obsoleted correctly - old packs and indices
727
        # in the obsolete_packs directory.
728
        large_pack_name = list(index.iter_all_entries())[0][1][0]
729
        # finally, committing again should not touch the large pack.
730
        tree.commit('commit not triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
731
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
732
        self.assertEqual(2, len(list(index.iter_all_entries())))
733
        pack_names = [node[1][0] for node in index.iter_all_entries()]
734
        self.assertTrue(large_pack_name in pack_names)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
735
736
3801.1.18 by Andrew Bennetts
Add a test that ensures that the autopack RPC is actually used for all pack formats.
737
class TestSmartServerAutopack(TestCaseWithTransport):
738
739
    def setUp(self):
740
        super(TestSmartServerAutopack, self).setUp()
741
        # Create a smart server that publishes whatever the backing VFS server
742
        # does.
743
        self.smart_server = server.SmartTCPServer_for_testing()
744
        self.smart_server.setUp(self.get_server())
745
        self.addCleanup(self.smart_server.tearDown)
746
        # Log all HPSS calls into self.hpss_calls.
747
        client._SmartClient.hooks.install_named_hook(
748
            'call', self.capture_hpss_call, None)
749
        self.hpss_calls = []
750
751
    def capture_hpss_call(self, params):
752
        self.hpss_calls.append(params.method)
753
754
    def get_format(self):
755
        return bzrdir.format_registry.make_bzrdir(self.format_name)
756
757
    def test_autopack_rpc_is_used_when_using_hpss(self):
758
        # Make local and remote repos
759
        tree = self.make_branch_and_tree('local', format=self.get_format())
760
        self.make_branch_and_tree('remote', format=self.get_format())
761
        remote_branch_url = self.smart_server.get_url() + 'remote'
762
        remote_branch = bzrdir.BzrDir.open(remote_branch_url).open_branch()
763
        # Make 9 local revisions, and push them one at a time to the remote
764
        # repo to produce 9 pack files.
765
        for x in range(9):
766
            tree.commit('commit %s' % x)
767
            tree.branch.push(remote_branch)
768
        # Make one more push to trigger an autopack
769
        self.hpss_calls = []
770
        tree.commit('commit triggering pack')
771
        tree.branch.push(remote_branch)
772
        self.assertTrue('PackRepository.autopack' in self.hpss_calls)
773
774
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
775
def load_tests(basic_tests, module, test_loader):
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
776
    # these give the bzrdir canned format name, and the repository on-disk
777
    # format string
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
778
    scenarios_params = [
779
         dict(format_name='pack-0.92',
780
              format_string="Bazaar pack repository format 1 (needs bzr 0.92)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
781
              format_supports_external_lookups=False,
782
              index_class=GraphIndex),
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
783
         dict(format_name='pack-0.92-subtree',
784
              format_string="Bazaar pack repository format 1 "
785
              "with subtree support (needs bzr 0.92)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
786
              format_supports_external_lookups=False,
787
              index_class=GraphIndex),
3582.3.2 by Martin Pool
Add 1.6 formats to pack repository tests
788
         dict(format_name='1.6',
789
              format_string="Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
790
              format_supports_external_lookups=True,
791
              index_class=GraphIndex),
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
792
         dict(format_name='1.6.1-rich-root',
3582.3.2 by Martin Pool
Add 1.6 formats to pack repository tests
793
              format_string="Bazaar RepositoryFormatKnitPack5RichRoot "
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
794
                  "(bzr 1.6.1)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
795
              format_supports_external_lookups=True,
796
              index_class=GraphIndex),
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
797
         dict(format_name='1.9',
798
              format_string="Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n",
799
              format_supports_external_lookups=True,
800
              index_class=BTreeGraphIndex),
801
         dict(format_name='1.9-rich-root',
802
              format_string="Bazaar RepositoryFormatKnitPack6RichRoot "
803
                  "(bzr 1.9)\n",
804
              format_supports_external_lookups=True,
805
              index_class=BTreeGraphIndex),
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
806
         dict(format_name='development2',
807
              format_string="Bazaar development format 2 "
808
                  "(needs bzr.dev from before 1.8)\n",
809
              format_supports_external_lookups=True,
810
              index_class=BTreeGraphIndex),
811
         dict(format_name='development2-subtree',
812
              format_string="Bazaar development format 2 "
813
                  "with subtree support (needs bzr.dev from before 1.8)\n",
814
              format_supports_external_lookups=True,
815
              index_class=BTreeGraphIndex),
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
816
         ]
817
    adapter = tests.TestScenarioApplier()
818
    # name of the scenario is the format name
819
    adapter.scenarios = [(s['format_name'], s) for s in scenarios_params]
820
    suite = tests.TestSuite()
821
    tests.adapt_tests(basic_tests, adapter, suite)
822
    return suite