/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
1
# Copyright (C) 2008 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Tests for pack repositories.
18
19
These tests are repeated for all pack-based repository formats.
20
"""
21
3582.3.4 by Martin Pool
Use cStringIO rather than StringIO
22
from cStringIO import StringIO
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
23
from stat import S_ISDIR
24
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
25
from bzrlib.btree_index import BTreeGraphIndex
26
from bzrlib.index import GraphIndex
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
27
from bzrlib import (
28
    bzrdir,
29
    errors,
30
    inventory,
31
    progress,
32
    repository,
33
    revision as _mod_revision,
34
    symbol_versioning,
35
    tests,
36
    ui,
37
    upgrade,
38
    workingtree,
39
    )
3801.1.18 by Andrew Bennetts
Add a test that ensures that the autopack RPC is actually used for all pack formats.
40
from bzrlib.smart import (
41
    client,
42
    server,
43
    )
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
44
from bzrlib.tests import (
45
    TestCase,
46
    TestCaseWithTransport,
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
47
    TestNotApplicable,
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
48
    TestSkipped,
49
    )
50
from bzrlib.transport import (
51
    fakenfs,
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
52
    memory,
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
53
    get_transport,
54
    )
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
55
from bzrlib.tests.per_repository import TestCaseWithRepository
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
56
57
58
class TestPackRepository(TestCaseWithTransport):
59
    """Tests to be repeated across all pack-based formats.
60
61
    The following are populated from the test scenario:
62
63
    :ivar format_name: Registered name fo the format to test.
64
    :ivar format_string: On-disk format marker.
65
    :ivar format_supports_external_lookups: Boolean.
66
    """
67
68
    def get_format(self):
69
        return bzrdir.format_registry.make_bzrdir(self.format_name)
70
71
    def test_attribute__fetch_order(self):
3606.7.3 by John Arbash Meinel
We don't have to fetch in topological order, as long as we fix all of the delta logic pieces.
72
        """Packs do not need ordered data retrieval."""
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
73
        format = self.get_format()
74
        repo = self.make_repository('.', format=format)
3606.7.8 by John Arbash Meinel
Switch names to 'unordered' that I missed before.
75
        self.assertEqual('unordered', repo._fetch_order)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
76
77
    def test_attribute__fetch_uses_deltas(self):
78
        """Packs reuse deltas."""
79
        format = self.get_format()
80
        repo = self.make_repository('.', format=format)
81
        self.assertEqual(True, repo._fetch_uses_deltas)
82
83
    def test_disk_layout(self):
84
        format = self.get_format()
85
        repo = self.make_repository('.', format=format)
86
        # in case of side effects of locking.
87
        repo.lock_write()
88
        repo.unlock()
89
        t = repo.bzrdir.get_repository_transport(None)
90
        self.check_format(t)
91
        # XXX: no locks left when unlocked at the moment
92
        # self.assertEqualDiff('', t.get('lock').read())
93
        self.check_databases(t)
94
95
    def check_format(self, t):
96
        self.assertEqualDiff(
97
            self.format_string, # from scenario
98
            t.get('format').read())
99
100
    def assertHasNoKndx(self, t, knit_name):
101
        """Assert that knit_name has no index on t."""
102
        self.assertFalse(t.has(knit_name + '.kndx'))
103
104
    def assertHasNoKnit(self, t, knit_name):
105
        """Assert that knit_name exists on t."""
106
        # no default content
107
        self.assertFalse(t.has(knit_name + '.knit'))
108
109
    def check_databases(self, t):
110
        """check knit content for a repository."""
111
        # check conversion worked
112
        self.assertHasNoKndx(t, 'inventory')
113
        self.assertHasNoKnit(t, 'inventory')
114
        self.assertHasNoKndx(t, 'revisions')
115
        self.assertHasNoKnit(t, 'revisions')
116
        self.assertHasNoKndx(t, 'signatures')
117
        self.assertHasNoKnit(t, 'signatures')
118
        self.assertFalse(t.has('knits'))
119
        # revision-indexes file-container directory
120
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
121
            list(self.index_class(t, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
122
        self.assertTrue(S_ISDIR(t.stat('packs').st_mode))
123
        self.assertTrue(S_ISDIR(t.stat('upload').st_mode))
124
        self.assertTrue(S_ISDIR(t.stat('indices').st_mode))
125
        self.assertTrue(S_ISDIR(t.stat('obsolete_packs').st_mode))
126
127
    def test_shared_disk_layout(self):
128
        format = self.get_format()
129
        repo = self.make_repository('.', shared=True, format=format)
130
        # we want:
131
        t = repo.bzrdir.get_repository_transport(None)
132
        self.check_format(t)
133
        # XXX: no locks left when unlocked at the moment
134
        # self.assertEqualDiff('', t.get('lock').read())
135
        # We should have a 'shared-storage' marker file.
136
        self.assertEqualDiff('', t.get('shared-storage').read())
137
        self.check_databases(t)
138
139
    def test_shared_no_tree_disk_layout(self):
140
        format = self.get_format()
141
        repo = self.make_repository('.', shared=True, format=format)
142
        repo.set_make_working_trees(False)
143
        # we want:
144
        t = repo.bzrdir.get_repository_transport(None)
145
        self.check_format(t)
146
        # XXX: no locks left when unlocked at the moment
147
        # self.assertEqualDiff('', t.get('lock').read())
148
        # We should have a 'shared-storage' marker file.
149
        self.assertEqualDiff('', t.get('shared-storage').read())
150
        # We should have a marker for the no-working-trees flag.
151
        self.assertEqualDiff('', t.get('no-working-trees').read())
152
        # The marker should go when we toggle the setting.
153
        repo.set_make_working_trees(True)
154
        self.assertFalse(t.has('no-working-trees'))
155
        self.check_databases(t)
156
157
    def test_adding_revision_creates_pack_indices(self):
158
        format = self.get_format()
159
        tree = self.make_branch_and_tree('.', format=format)
160
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
161
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
162
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
163
        tree.commit('foobarbaz')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
164
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
165
        index_nodes = list(index.iter_all_entries())
166
        self.assertEqual(1, len(index_nodes))
167
        node = index_nodes[0]
168
        name = node[1][0]
169
        # the pack sizes should be listed in the index
170
        pack_value = node[2]
171
        sizes = [int(digits) for digits in pack_value.split(' ')]
172
        for size, suffix in zip(sizes, ['.rix', '.iix', '.tix', '.six']):
173
            stat = trans.stat('indices/%s%s' % (name, suffix))
174
            self.assertEqual(size, stat.st_size)
175
176
    def test_pulling_nothing_leads_to_no_new_names(self):
177
        format = self.get_format()
178
        tree1 = self.make_branch_and_tree('1', format=format)
179
        tree2 = self.make_branch_and_tree('2', format=format)
180
        tree1.branch.repository.fetch(tree2.branch.repository)
181
        trans = tree1.branch.repository.bzrdir.get_repository_transport(None)
182
        self.assertEqual([],
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
183
            list(self.index_class(trans, 'pack-names', None).iter_all_entries()))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
184
185
    def test_commit_across_pack_shape_boundary_autopacks(self):
186
        format = self.get_format()
187
        tree = self.make_branch_and_tree('.', format=format)
188
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
189
        # This test could be a little cheaper by replacing the packs
190
        # attribute on the repository to allow a different pack distribution
191
        # and max packs policy - so we are checking the policy is honoured
192
        # in the test. But for now 11 commits is not a big deal in a single
193
        # test.
194
        for x in range(9):
195
            tree.commit('commit %s' % x)
196
        # there should be 9 packs:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
197
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
198
        self.assertEqual(9, len(list(index.iter_all_entries())))
199
        # insert some files in obsolete_packs which should be removed by pack.
200
        trans.put_bytes('obsolete_packs/foo', '123')
201
        trans.put_bytes('obsolete_packs/bar', '321')
202
        # committing one more should coalesce to 1 of 10.
203
        tree.commit('commit triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
204
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
205
        self.assertEqual(1, len(list(index.iter_all_entries())))
206
        # packing should not damage data
207
        tree = tree.bzrdir.open_workingtree()
208
        check_result = tree.branch.repository.check(
209
            [tree.branch.last_revision()])
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
210
        nb_files = 5 # .pack, .rix, .iix, .tix, .six
211
        if tree.branch.repository._format.supports_chks:
212
            nb_files += 1 # .cix
213
        # We should have 10 x nb_files files in the obsolete_packs directory.
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
214
        obsolete_files = list(trans.list_dir('obsolete_packs'))
215
        self.assertFalse('foo' in obsolete_files)
216
        self.assertFalse('bar' in obsolete_files)
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
217
        self.assertEqual(10 * nb_files, len(obsolete_files))
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
218
        # XXX: Todo check packs obsoleted correctly - old packs and indices
219
        # in the obsolete_packs directory.
220
        large_pack_name = list(index.iter_all_entries())[0][1][0]
221
        # finally, committing again should not touch the large pack.
222
        tree.commit('commit not triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
223
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
224
        self.assertEqual(2, len(list(index.iter_all_entries())))
225
        pack_names = [node[1][0] for node in index.iter_all_entries()]
226
        self.assertTrue(large_pack_name in pack_names)
227
228
    def test_fail_obsolete_deletion(self):
229
        # failing to delete obsolete packs is not fatal
230
        format = self.get_format()
231
        server = fakenfs.FakeNFSServer()
232
        server.setUp()
233
        self.addCleanup(server.tearDown)
234
        transport = get_transport(server.get_url())
235
        bzrdir = self.get_format().initialize_on_transport(transport)
236
        repo = bzrdir.create_repository()
237
        repo_transport = bzrdir.get_repository_transport(None)
238
        self.assertTrue(repo_transport.has('obsolete_packs'))
239
        # these files are in use by another client and typically can't be deleted
240
        repo_transport.put_bytes('obsolete_packs/.nfsblahblah', 'contents')
241
        repo._pack_collection._clear_obsolete_packs()
242
        self.assertTrue(repo_transport.has('obsolete_packs/.nfsblahblah'))
243
244
    def test_pack_after_two_commits_packs_everything(self):
245
        format = self.get_format()
246
        tree = self.make_branch_and_tree('.', format=format)
247
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
248
        tree.commit('start')
249
        tree.commit('more work')
250
        tree.branch.repository.pack()
251
        # there should be 1 pack:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
252
        index = self.index_class(trans, 'pack-names', None)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
253
        self.assertEqual(1, len(list(index.iter_all_entries())))
254
        self.assertEqual(2, len(tree.branch.repository.all_revision_ids()))
255
256
    def test_pack_layout(self):
257
        format = self.get_format()
258
        tree = self.make_branch_and_tree('.', format=format)
259
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
260
        tree.commit('start', rev_id='1')
261
        tree.commit('more work', rev_id='2')
262
        tree.branch.repository.pack()
263
        tree.lock_read()
264
        self.addCleanup(tree.unlock)
265
        pack = tree.branch.repository._pack_collection.get_pack_by_name(
266
            tree.branch.repository._pack_collection.names()[0])
267
        # revision access tends to be tip->ancestor, so ordering that way on 
268
        # disk is a good idea.
269
        for _1, key, val, refs in pack.revision_index.iter_all_entries():
270
            if key == ('1',):
271
                pos_1 = int(val[1:].split()[0])
272
            else:
273
                pos_2 = int(val[1:].split()[0])
274
        self.assertTrue(pos_2 < pos_1)
275
276
    def test_pack_repositories_support_multiple_write_locks(self):
277
        format = self.get_format()
278
        self.make_repository('.', shared=True, format=format)
279
        r1 = repository.Repository.open('.')
280
        r2 = repository.Repository.open('.')
281
        r1.lock_write()
282
        self.addCleanup(r1.unlock)
283
        r2.lock_write()
284
        r2.unlock()
285
286
    def _add_text(self, repo, fileid):
287
        """Add a text to the repository within a write group."""
288
        repo.texts.add_lines((fileid, 'samplerev+'+fileid), [], [])
289
290
    def test_concurrent_writers_merge_new_packs(self):
291
        format = self.get_format()
292
        self.make_repository('.', shared=True, format=format)
293
        r1 = repository.Repository.open('.')
294
        r2 = repository.Repository.open('.')
295
        r1.lock_write()
296
        try:
297
            # access enough data to load the names list
298
            list(r1.all_revision_ids())
299
            r2.lock_write()
300
            try:
301
                # access enough data to load the names list
302
                list(r2.all_revision_ids())
303
                r1.start_write_group()
304
                try:
305
                    r2.start_write_group()
306
                    try:
307
                        self._add_text(r1, 'fileidr1')
308
                        self._add_text(r2, 'fileidr2')
309
                    except:
310
                        r2.abort_write_group()
311
                        raise
312
                except:
313
                    r1.abort_write_group()
314
                    raise
315
                # both r1 and r2 have open write groups with data in them
316
                # created while the other's write group was open.
317
                # Commit both which requires a merge to the pack-names.
318
                try:
319
                    r1.commit_write_group()
320
                except:
321
                    r1.abort_write_group()
322
                    r2.abort_write_group()
323
                    raise
324
                r2.commit_write_group()
325
                # tell r1 to reload from disk
326
                r1._pack_collection.reset()
327
                # Now both repositories should know about both names
328
                r1._pack_collection.ensure_loaded()
329
                r2._pack_collection.ensure_loaded()
330
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
331
                self.assertEqual(2, len(r1._pack_collection.names()))
332
            finally:
333
                r2.unlock()
334
        finally:
335
            r1.unlock()
336
337
    def test_concurrent_writer_second_preserves_dropping_a_pack(self):
338
        format = self.get_format()
339
        self.make_repository('.', shared=True, format=format)
340
        r1 = repository.Repository.open('.')
341
        r2 = repository.Repository.open('.')
342
        # add a pack to drop
343
        r1.lock_write()
344
        try:
345
            r1.start_write_group()
346
            try:
347
                self._add_text(r1, 'fileidr1')
348
            except:
349
                r1.abort_write_group()
350
                raise
351
            else:
352
                r1.commit_write_group()
353
            r1._pack_collection.ensure_loaded()
354
            name_to_drop = r1._pack_collection.all_packs()[0].name
355
        finally:
356
            r1.unlock()
357
        r1.lock_write()
358
        try:
359
            # access enough data to load the names list
360
            list(r1.all_revision_ids())
361
            r2.lock_write()
362
            try:
363
                # access enough data to load the names list
364
                list(r2.all_revision_ids())
365
                r1._pack_collection.ensure_loaded()
366
                try:
367
                    r2.start_write_group()
368
                    try:
369
                        # in r1, drop the pack
370
                        r1._pack_collection._remove_pack_from_memory(
371
                            r1._pack_collection.get_pack_by_name(name_to_drop))
372
                        # in r2, add a pack
373
                        self._add_text(r2, 'fileidr2')
374
                    except:
375
                        r2.abort_write_group()
376
                        raise
377
                except:
378
                    r1._pack_collection.reset()
379
                    raise
380
                # r1 has a changed names list, and r2 an open write groups with
381
                # changes.
382
                # save r1, and then commit the r2 write group, which requires a
383
                # merge to the pack-names, which should not reinstate
384
                # name_to_drop
385
                try:
386
                    r1._pack_collection._save_pack_names()
387
                    r1._pack_collection.reset()
388
                except:
389
                    r2.abort_write_group()
390
                    raise
391
                try:
392
                    r2.commit_write_group()
393
                except:
394
                    r2.abort_write_group()
395
                    raise
396
                # Now both repositories should now about just one name.
397
                r1._pack_collection.ensure_loaded()
398
                r2._pack_collection.ensure_loaded()
399
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
400
                self.assertEqual(1, len(r1._pack_collection.names()))
401
                self.assertFalse(name_to_drop in r1._pack_collection.names())
402
            finally:
403
                r2.unlock()
404
        finally:
405
            r1.unlock()
406
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
407
    def test_concurrent_pack_triggers_reload(self):
408
        # create 2 packs, which we will then collapse
409
        tree = self.make_branch_and_tree('tree')
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
410
        tree.lock_write()
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
411
        try:
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
412
            rev1 = tree.commit('one')
413
            rev2 = tree.commit('two')
414
            r2 = repository.Repository.open('tree')
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
415
            r2.lock_read()
416
            try:
417
                # Now r2 has read the pack-names file, but will need to reload
418
                # it after r1 has repacked
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
419
                tree.branch.repository.pack()
420
                self.assertEqual({rev2:(rev1,)}, r2.get_parent_map([rev2]))
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
421
            finally:
422
                r2.unlock()
423
        finally:
3789.1.2 by John Arbash Meinel
Add RepositoryPackCollection.reload_pack_names()
424
            tree.unlock()
3789.1.1 by John Arbash Meinel
add the failing acceptance test for the first portion.
425
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
426
    def test_concurrent_pack_during_get_record_reloads(self):
427
        tree = self.make_branch_and_tree('tree')
428
        tree.lock_write()
429
        try:
430
            rev1 = tree.commit('one')
431
            rev2 = tree.commit('two')
3789.2.14 by John Arbash Meinel
Update AggregateIndex to pass the reload_func into _DirectPackAccess
432
            keys = [(rev1,), (rev2,)]
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
433
            r2 = repository.Repository.open('tree')
434
            r2.lock_read()
435
            try:
436
                # At this point, we will start grabbing a record stream, and
437
                # trigger a repack mid-way
438
                packed = False
439
                result = {}
440
                record_stream = r2.revisions.get_record_stream(keys,
441
                                    'unordered', False)
442
                for record in record_stream:
443
                    result[record.key] = record
444
                    if not packed:
445
                        tree.branch.repository.pack()
446
                        packed = True
447
                # The first record will be found in the original location, but
448
                # after the pack, we have to reload to find the next record
3789.2.14 by John Arbash Meinel
Update AggregateIndex to pass the reload_func into _DirectPackAccess
449
                self.assertEqual(sorted(keys), sorted(result.keys()))
3789.2.8 by John Arbash Meinel
Add a test that KnitPackRepository.get_record_stream retries when appropriate.
450
            finally:
451
                r2.unlock()
452
        finally:
453
            tree.unlock()
454
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
455
    def test_lock_write_does_not_physically_lock(self):
456
        repo = self.make_repository('.', format=self.get_format())
457
        repo.lock_write()
458
        self.addCleanup(repo.unlock)
459
        self.assertFalse(repo.get_physical_lock_status())
460
461
    def prepare_for_break_lock(self):
462
        # Setup the global ui factory state so that a break-lock method call
463
        # will find usable input in the input stream.
464
        old_factory = ui.ui_factory
465
        def restoreFactory():
466
            ui.ui_factory = old_factory
467
        self.addCleanup(restoreFactory)
468
        ui.ui_factory = ui.SilentUIFactory()
469
        ui.ui_factory.stdin = StringIO("y\n")
470
471
    def test_break_lock_breaks_physical_lock(self):
472
        repo = self.make_repository('.', format=self.get_format())
473
        repo._pack_collection.lock_names()
3650.4.1 by Aaron Bentley
Fix test kipple in test_break_lock_breaks_physical_lock
474
        repo.control_files.leave_in_place()
475
        repo.unlock()
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
476
        repo2 = repository.Repository.open('.')
477
        self.assertTrue(repo.get_physical_lock_status())
478
        self.prepare_for_break_lock()
479
        repo2.break_lock()
480
        self.assertFalse(repo.get_physical_lock_status())
481
482
    def test_broken_physical_locks_error_on__unlock_names_lock(self):
483
        repo = self.make_repository('.', format=self.get_format())
484
        repo._pack_collection.lock_names()
485
        self.assertTrue(repo.get_physical_lock_status())
486
        repo2 = repository.Repository.open('.')
487
        self.prepare_for_break_lock()
488
        repo2.break_lock()
489
        self.assertRaises(errors.LockBroken, repo._pack_collection._unlock_names)
490
491
    def test_fetch_without_find_ghosts_ignores_ghosts(self):
492
        # we want two repositories at this point:
493
        # one with a revision that is a ghost in the other
494
        # repository.
495
        # 'ghost' is present in has_ghost, 'ghost' is absent in 'missing_ghost'.
496
        # 'references' is present in both repositories, and 'tip' is present
497
        # just in has_ghost.
498
        # has_ghost       missing_ghost
499
        #------------------------------
500
        # 'ghost'             -
501
        # 'references'    'references'
502
        # 'tip'               -
503
        # In this test we fetch 'tip' which should not fetch 'ghost'
504
        has_ghost = self.make_repository('has_ghost', format=self.get_format())
505
        missing_ghost = self.make_repository('missing_ghost',
506
            format=self.get_format())
507
508
        def add_commit(repo, revision_id, parent_ids):
509
            repo.lock_write()
510
            repo.start_write_group()
511
            inv = inventory.Inventory(revision_id=revision_id)
512
            inv.root.revision = revision_id
513
            root_id = inv.root.file_id
514
            sha1 = repo.add_inventory(revision_id, inv, [])
515
            repo.texts.add_lines((root_id, revision_id), [], [])
516
            rev = _mod_revision.Revision(timestamp=0,
517
                                         timezone=None,
518
                                         committer="Foo Bar <foo@example.com>",
519
                                         message="Message",
520
                                         inventory_sha1=sha1,
521
                                         revision_id=revision_id)
522
            rev.parent_ids = parent_ids
523
            repo.add_revision(revision_id, rev)
524
            repo.commit_write_group()
525
            repo.unlock()
526
        add_commit(has_ghost, 'ghost', [])
527
        add_commit(has_ghost, 'references', ['ghost'])
528
        add_commit(missing_ghost, 'references', ['ghost'])
529
        add_commit(has_ghost, 'tip', ['references'])
530
        missing_ghost.fetch(has_ghost, 'tip')
531
        # missing ghost now has tip and not ghost.
532
        rev = missing_ghost.get_revision('tip')
533
        inv = missing_ghost.get_inventory('tip')
534
        self.assertRaises(errors.NoSuchRevision,
535
            missing_ghost.get_revision, 'ghost')
536
        self.assertRaises(errors.NoSuchRevision,
537
            missing_ghost.get_inventory, 'ghost')
538
539
    def test_supports_external_lookups(self):
540
        repo = self.make_repository('.', format=self.get_format())
541
        self.assertEqual(self.format_supports_external_lookups,
542
            repo._format.supports_external_lookups)
543
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
544
    def test_abort_write_group_does_not_raise_when_suppressed(self):
545
        """Similar to per_repository.test_write_group's test of the same name.
546
547
        Also requires that the exception is logged.
548
        """
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
549
        self.vfs_transport_factory = memory.MemoryServer
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
550
        repo = self.make_repository('repo')
551
        token = repo.lock_write()
552
        self.addCleanup(repo.unlock)
553
        repo.start_write_group()
554
        # Damage the repository on the filesystem
555
        self.get_transport('').rename('repo', 'foo')
556
        # abort_write_group will not raise an error
557
        self.assertEqual(None, repo.abort_write_group(suppress_errors=True))
558
        # But it does log an error
559
        log_file = self._get_log(keep_log_file=True)
560
        self.assertContainsRe(log_file, 'abort_write_group failed')
561
        self.assertContainsRe(log_file, r'INFO  bzr: ERROR \(ignored\):')
562
        if token is not None:
563
            repo.leave_lock_in_place()
564
        
565
    def test_abort_write_group_does_raise_when_not_suppressed(self):
3825.4.2 by Andrew Bennetts
Run the abort_write_group tests against a memory transport to avoid platform-specific limits on changing files that may be in use.
566
        self.vfs_transport_factory = memory.MemoryServer
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
567
        repo = self.make_repository('repo')
568
        token = repo.lock_write()
569
        self.addCleanup(repo.unlock)
570
        repo.start_write_group()
571
        # Damage the repository on the filesystem
572
        self.get_transport('').rename('repo', 'foo')
573
        # abort_write_group will not raise an error
574
        self.assertRaises(Exception, repo.abort_write_group)
575
        if token is not None:
576
            repo.leave_lock_in_place()
577
        
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
578
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
579
class TestPackRepositoryStacking(TestCaseWithTransport):
580
581
    """Tests for stacking pack repositories"""
582
583
    def setUp(self):
584
        if not self.format_supports_external_lookups:
585
            raise TestNotApplicable("%r doesn't support stacking" 
586
                % (self.format_name,))
587
        super(TestPackRepositoryStacking, self).setUp()
588
589
    def get_format(self):
590
        return bzrdir.format_registry.make_bzrdir(self.format_name)
591
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
592
    def test_stack_checks_rich_root_compatibility(self):
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
593
        # early versions of the packing code relied on pack internals to
594
        # stack, but the current version should be able to stack on any
595
        # format.
596
        #
597
        # TODO: Possibly this should be run per-repository-format and raise
598
        # TestNotApplicable on formats that don't support stacking. -- mbp
599
        # 20080729
600
        repo = self.make_repository('repo', format=self.get_format())
601
        if repo.supports_rich_root():
602
            # can only stack on repositories that have compatible internal
603
            # metadata
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
604
            if getattr(repo._format, 'supports_tree_reference', False):
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
605
                if repo._format.supports_chks:
606
                    matching_format_name = 'development3-subtree'
607
                else:
608
                    matching_format_name = 'pack-0.92-subtree'
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
609
            else:
610
                matching_format_name = 'rich-root-pack'
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
611
            mismatching_format_name = 'pack-0.92'
612
        else:
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
613
            if repo._format.supports_chks:
614
                matching_format_name = 'development3'
615
            else:
616
                matching_format_name = 'pack-0.92'
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
617
            mismatching_format_name = 'pack-0.92-subtree'
618
        base = self.make_repository('base', format=matching_format_name)
619
        repo.add_fallback_repository(base)
620
        # you can't stack on something with incompatible data
621
        bad_repo = self.make_repository('mismatch',
622
            format=mismatching_format_name)
623
        e = self.assertRaises(errors.IncompatibleRepositories,
624
            repo.add_fallback_repository, bad_repo)
625
        self.assertContainsRe(str(e),
626
            r'(?m)KnitPackRepository.*/mismatch/.*\nis not compatible with\n'
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
627
            r'.*Repository.*/repo/.*\n'
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
628
            r'different rich-root support')
629
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
630
    def test_stack_checks_serializers_compatibility(self):
631
        repo = self.make_repository('repo', format=self.get_format())
632
        if getattr(repo._format, 'supports_tree_reference', False):
633
            # can only stack on repositories that have compatible internal
634
            # metadata
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
635
            if repo._format.supports_chks:
636
                matching_format_name = 'development3-subtree'
637
            else:
638
                matching_format_name = 'pack-0.92-subtree'
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
639
            mismatching_format_name = 'rich-root-pack'
640
        else:
641
            if repo.supports_rich_root():
642
                matching_format_name = 'rich-root-pack'
643
                mismatching_format_name = 'pack-0.92-subtree'
644
            else:
645
                raise TestNotApplicable('No formats use non-v5 serializer'
646
                    ' without having rich-root also set')
647
        base = self.make_repository('base', format=matching_format_name)
648
        repo.add_fallback_repository(base)
649
        # you can't stack on something with incompatible data
650
        bad_repo = self.make_repository('mismatch',
651
            format=mismatching_format_name)
652
        e = self.assertRaises(errors.IncompatibleRepositories,
653
            repo.add_fallback_repository, bad_repo)
654
        self.assertContainsRe(str(e),
655
            r'(?m)KnitPackRepository.*/mismatch/.*\nis not compatible with\n'
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
656
            r'.*Repository.*/repo/.*\n'
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
657
            r'different serializers')
658
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
659
    def test_adding_pack_does_not_record_pack_names_from_other_repositories(self):
660
        base = self.make_branch_and_tree('base', format=self.get_format())
661
        base.commit('foo')
662
        referencing = self.make_branch_and_tree('repo', format=self.get_format())
663
        referencing.branch.repository.add_fallback_repository(base.branch.repository)
664
        referencing.commit('bar')
665
        new_instance = referencing.bzrdir.open_repository()
666
        new_instance.lock_read()
667
        self.addCleanup(new_instance.unlock)
668
        new_instance._pack_collection.ensure_loaded()
669
        self.assertEqual(1, len(new_instance._pack_collection.all_packs()))
670
671
    def test_autopack_only_considers_main_repo_packs(self):
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
672
        format = self.get_format()
673
        base = self.make_branch_and_tree('base', format=format)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
674
        base.commit('foo')
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
675
        tree = self.make_branch_and_tree('repo', format=format)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
676
        tree.branch.repository.add_fallback_repository(base.branch.repository)
677
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
678
        # This test could be a little cheaper by replacing the packs
679
        # attribute on the repository to allow a different pack distribution
680
        # and max packs policy - so we are checking the policy is honoured
681
        # in the test. But for now 11 commits is not a big deal in a single
682
        # test.
683
        for x in range(9):
684
            tree.commit('commit %s' % x)
685
        # there should be 9 packs:
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
686
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
687
        self.assertEqual(9, len(list(index.iter_all_entries())))
688
        # committing one more should coalesce to 1 of 10.
689
        tree.commit('commit triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
690
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
691
        self.assertEqual(1, len(list(index.iter_all_entries())))
692
        # packing should not damage data
693
        tree = tree.bzrdir.open_workingtree()
694
        check_result = tree.branch.repository.check(
695
            [tree.branch.last_revision()])
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
696
        nb_files = 5 # .pack, .rix, .iix, .tix, .six
697
        if tree.branch.repository._format.supports_chks:
698
            nb_files += 1 # .cix
699
        # We should have 10 x nb_files files in the obsolete_packs directory.
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
700
        obsolete_files = list(trans.list_dir('obsolete_packs'))
701
        self.assertFalse('foo' in obsolete_files)
702
        self.assertFalse('bar' in obsolete_files)
3735.2.81 by Vincent Ladeuil
Fix test failing on files present in obsolete_packs directory.
703
        self.assertEqual(10 * nb_files, len(obsolete_files))
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
704
        # XXX: Todo check packs obsoleted correctly - old packs and indices
705
        # in the obsolete_packs directory.
706
        large_pack_name = list(index.iter_all_entries())[0][1][0]
707
        # finally, committing again should not touch the large pack.
708
        tree.commit('commit not triggering pack')
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
709
        index = self.index_class(trans, 'pack-names', None)
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
710
        self.assertEqual(2, len(list(index.iter_all_entries())))
711
        pack_names = [node[1][0] for node in index.iter_all_entries()]
712
        self.assertTrue(large_pack_name in pack_names)
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
713
714
3801.1.18 by Andrew Bennetts
Add a test that ensures that the autopack RPC is actually used for all pack formats.
715
class TestSmartServerAutopack(TestCaseWithTransport):
716
717
    def setUp(self):
718
        super(TestSmartServerAutopack, self).setUp()
719
        # Create a smart server that publishes whatever the backing VFS server
720
        # does.
721
        self.smart_server = server.SmartTCPServer_for_testing()
722
        self.smart_server.setUp(self.get_server())
723
        self.addCleanup(self.smart_server.tearDown)
724
        # Log all HPSS calls into self.hpss_calls.
725
        client._SmartClient.hooks.install_named_hook(
726
            'call', self.capture_hpss_call, None)
727
        self.hpss_calls = []
728
729
    def capture_hpss_call(self, params):
730
        self.hpss_calls.append(params.method)
731
732
    def get_format(self):
733
        return bzrdir.format_registry.make_bzrdir(self.format_name)
734
735
    def test_autopack_rpc_is_used_when_using_hpss(self):
736
        # Make local and remote repos
737
        tree = self.make_branch_and_tree('local', format=self.get_format())
738
        self.make_branch_and_tree('remote', format=self.get_format())
739
        remote_branch_url = self.smart_server.get_url() + 'remote'
740
        remote_branch = bzrdir.BzrDir.open(remote_branch_url).open_branch()
741
        # Make 9 local revisions, and push them one at a time to the remote
742
        # repo to produce 9 pack files.
743
        for x in range(9):
744
            tree.commit('commit %s' % x)
745
            tree.branch.push(remote_branch)
746
        # Make one more push to trigger an autopack
747
        self.hpss_calls = []
748
        tree.commit('commit triggering pack')
749
        tree.branch.push(remote_branch)
750
        self.assertTrue('PackRepository.autopack' in self.hpss_calls)
751
752
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
753
def load_tests(basic_tests, module, test_loader):
3582.3.3 by Martin Pool
Reenable tests for stacking pack repositories
754
    # these give the bzrdir canned format name, and the repository on-disk
755
    # format string
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
756
    scenarios_params = [
757
         dict(format_name='pack-0.92',
758
              format_string="Bazaar pack repository format 1 (needs bzr 0.92)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
759
              format_supports_external_lookups=False,
760
              index_class=GraphIndex),
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
761
         dict(format_name='pack-0.92-subtree',
762
              format_string="Bazaar pack repository format 1 "
763
              "with subtree support (needs bzr 0.92)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
764
              format_supports_external_lookups=False,
765
              index_class=GraphIndex),
3582.3.2 by Martin Pool
Add 1.6 formats to pack repository tests
766
         dict(format_name='1.6',
767
              format_string="Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
768
              format_supports_external_lookups=True,
769
              index_class=GraphIndex),
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
770
         dict(format_name='1.6.1-rich-root',
3582.3.2 by Martin Pool
Add 1.6 formats to pack repository tests
771
              format_string="Bazaar RepositoryFormatKnitPack5RichRoot "
3606.10.5 by John Arbash Meinel
Switch out --1.6-rich-root for --1.6.1-rich-root.
772
                  "(bzr 1.6.1)\n",
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
773
              format_supports_external_lookups=True,
774
              index_class=GraphIndex),
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
775
         dict(format_name='1.9',
776
              format_string="Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n",
777
              format_supports_external_lookups=True,
778
              index_class=BTreeGraphIndex),
779
         dict(format_name='1.9-rich-root',
780
              format_string="Bazaar RepositoryFormatKnitPack6RichRoot "
781
                  "(bzr 1.9)\n",
782
              format_supports_external_lookups=True,
783
              index_class=BTreeGraphIndex),
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
784
         dict(format_name='development2',
785
              format_string="Bazaar development format 2 "
786
                  "(needs bzr.dev from before 1.8)\n",
787
              format_supports_external_lookups=True,
788
              index_class=BTreeGraphIndex),
789
         dict(format_name='development2-subtree',
790
              format_string="Bazaar development format 2 "
791
                  "with subtree support (needs bzr.dev from before 1.8)\n",
792
              format_supports_external_lookups=True,
793
              index_class=BTreeGraphIndex),
3735.2.3 by Robert Collins
Create a format which passes the basic smoke test for CHK availability.
794
         dict(format_name='development3',
795
              format_string="Bazaar development format 3 "
3735.6.1 by Vincent Ladeuil
Reduce failing tests from 62 errors/21 failures to 6 errors/3 failures.
796
                  "(needs bzr.dev from before 1.10)\n",
3735.2.3 by Robert Collins
Create a format which passes the basic smoke test for CHK availability.
797
              format_supports_external_lookups=True,
798
              index_class=BTreeGraphIndex),
799
         dict(format_name='development3-subtree',
800
              format_string="Bazaar development format 3 "
3735.6.1 by Vincent Ladeuil
Reduce failing tests from 62 errors/21 failures to 6 errors/3 failures.
801
                  "with subtree support (needs bzr.dev from before 1.10)\n",
3735.2.3 by Robert Collins
Create a format which passes the basic smoke test for CHK availability.
802
              format_supports_external_lookups=True,
803
              index_class=BTreeGraphIndex),
3735.2.40 by Robert Collins
Add development4 which has a parent_id to basename index on CHKInventory objects.
804
         dict(format_name='development4',
3735.2.82 by Vincent Ladeuil
Cough, stop harassing early testers.
805
              # merge-bbc-dev4-to-bzr.dev
3735.2.40 by Robert Collins
Add development4 which has a parent_id to basename index on CHKInventory objects.
806
              format_string="Bazaar development format 4 "
3735.2.82 by Vincent Ladeuil
Cough, stop harassing early testers.
807
                  "(needs bzr.dev from before 1.11)\n",
3735.2.40 by Robert Collins
Add development4 which has a parent_id to basename index on CHKInventory objects.
808
              format_supports_external_lookups=True,
809
              index_class=BTreeGraphIndex),
810
         dict(format_name='development4-subtree',
3735.2.82 by Vincent Ladeuil
Cough, stop harassing early testers.
811
              # merge-bbc-dev4-to-bzr.dev
3735.2.40 by Robert Collins
Add development4 which has a parent_id to basename index on CHKInventory objects.
812
              format_string="Bazaar development format 4 "
3735.2.82 by Vincent Ladeuil
Cough, stop harassing early testers.
813
                  "with subtree support (needs bzr.dev from before 1.11)\n",
3735.2.40 by Robert Collins
Add development4 which has a parent_id to basename index on CHKInventory objects.
814
              format_supports_external_lookups=True,
815
              index_class=BTreeGraphIndex),
3582.3.1 by Martin Pool
Split pack repository tests into their own file and use scenarios
816
         ]
817
    adapter = tests.TestScenarioApplier()
818
    # name of the scenario is the format name
819
    adapter.scenarios = [(s['format_name'], s) for s in scenarios_params]
820
    suite = tests.TestSuite()
821
    tests.adapt_tests(basic_tests, adapter, suite)
822
    return suite