/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
2052.3.2 by John Arbash Meinel
Change Copyright .. by Canonical to Copyright ... Canonical
1
# Copyright (C) 2005 Canonical Ltd
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
2
#
3
# Authors:
4
#   Johan Rydberg <jrydberg@gnu.org>
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
10
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
15
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19
20
1704.2.15 by Martin Pool
Remove TODO about knit testing printed from test suite
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
23
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
24
from itertools import chain, izip
1664.2.9 by Aaron Bentley
Ported weave merge test to versionedfile
25
from StringIO import StringIO
26
1563.2.6 by Robert Collins
Start check tests for knits (pending), and remove dead code.
27
import bzrlib
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
28
from bzrlib import (
29
    errors,
2309.4.7 by John Arbash Meinel
Update VersionedFile tests to ensure that they can take Unicode,
30
    osutils,
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
31
    progress,
32
    )
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
33
from bzrlib.errors import (
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
34
                           RevisionNotPresent,
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
35
                           RevisionAlreadyPresent,
36
                           WeaveParentMismatch
37
                           )
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
38
from bzrlib import knit as _mod_knit
2770.1.1 by Aaron Bentley
Initial implmentation of plain knit annotation
39
from bzrlib.knit import (
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
40
    cleanup_pack_knit,
41
    make_file_factory,
42
    make_pack_factory,
2770.1.1 by Aaron Bentley
Initial implmentation of plain knit annotation
43
    KnitAnnotateFactory,
2770.1.10 by Aaron Bentley
Merge bzr.dev
44
    KnitPlainFactory,
2770.1.1 by Aaron Bentley
Initial implmentation of plain knit annotation
45
    )
3350.3.14 by Robert Collins
Deprecate VersionedFile.join.
46
from bzrlib.symbol_versioning import one_four, one_five
3350.6.2 by Robert Collins
Prepare parameterised test environment.
47
from bzrlib.tests import (
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
48
    TestCase,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
49
    TestCaseWithMemoryTransport,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
50
    TestNotApplicable,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
51
    TestScenarioApplier,
52
    TestSkipped,
53
    condition_isinstance,
54
    split_suite_by_condition,
55
    iter_suite_tests,
56
    )
2929.3.8 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix uses.
57
from bzrlib.tests.http_utils import TestCaseWithWebserver
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
58
from bzrlib.trace import mutter
1563.2.16 by Robert Collins
Change WeaveStore into VersionedFileStore and make its versoined file class parameterisable.
59
from bzrlib.transport import get_transport
1563.2.13 by Robert Collins
InterVersionedFile implemented.
60
from bzrlib.transport.memory import MemoryTransport
1684.3.1 by Robert Collins
Fix versioned file joins with empty targets.
61
from bzrlib.tsort import topo_sort
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
62
from bzrlib.tuned_gzip import GzipFile
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
63
import bzrlib.versionedfile as versionedfile
3350.6.2 by Robert Collins
Prepare parameterised test environment.
64
from bzrlib.versionedfile import (
65
    ConstantMapper,
66
    HashEscapedPrefixMapper,
67
    PrefixMapper,
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
68
    VirtualVersionedFiles,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
69
    make_versioned_files_factory,
70
    )
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
71
from bzrlib.weave import WeaveFile
1664.2.9 by Aaron Bentley
Ported weave merge test to versionedfile
72
from bzrlib.weavefile import read_weave, write_weave
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
73
74
3350.6.2 by Robert Collins
Prepare parameterised test environment.
75
def load_tests(standard_tests, module, loader):
76
    """Parameterize VersionedFiles tests for different implementations."""
77
    to_adapt, result = split_suite_by_condition(
78
        standard_tests, condition_isinstance(TestVersionedFiles))
79
    len_one_adapter = TestScenarioApplier()
80
    len_two_adapter = TestScenarioApplier()
81
    # We want to be sure of behaviour for:
82
    # weaves prefix layout (weave texts)
83
    # individually named weaves (weave inventories)
84
    # annotated knits - prefix|hash|hash-escape layout, we test the third only
85
    #                   as it is the most complex mapper.
86
    # individually named knits
87
    # individual no-graph knits in packs (signatures)
88
    # individual graph knits in packs (inventories)
89
    # individual graph nocompression knits in packs (revisions)
90
    # plain text knits in packs (texts)
91
    len_one_adapter.scenarios = [
92
        ('weave-named', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
93
            'cleanup':None,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
94
            'factory':make_versioned_files_factory(WeaveFile,
95
                ConstantMapper('inventory')),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
96
            'graph':True,
97
            'key_length':1,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
98
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
99
            }),
100
        ('named-knit', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
101
            'cleanup':None,
102
            'factory':make_file_factory(False, ConstantMapper('revisions')),
103
            'graph':True,
104
            'key_length':1,
4009.3.7 by Andrew Bennetts
Most tests passing.
105
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
106
            }),
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
107
        ('named-nograph-nodelta-knit-pack', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
108
            'cleanup':cleanup_pack_knit,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
109
            'factory':make_pack_factory(False, False, 1),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
110
            'graph':False,
111
            'key_length':1,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
112
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
113
            }),
114
        ('named-graph-knit-pack', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
115
            'cleanup':cleanup_pack_knit,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
116
            'factory':make_pack_factory(True, True, 1),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
117
            'graph':True,
118
            'key_length':1,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
119
            'support_partial_insertion': True,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
120
            }),
121
        ('named-graph-nodelta-knit-pack', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
122
            'cleanup':cleanup_pack_knit,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
123
            'factory':make_pack_factory(True, False, 1),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
124
            'graph':True,
125
            'key_length':1,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
126
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
127
            }),
128
        ]
129
    len_two_adapter.scenarios = [
130
        ('weave-prefix', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
131
            'cleanup':None,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
132
            'factory':make_versioned_files_factory(WeaveFile,
133
                PrefixMapper()),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
134
            'graph':True,
135
            'key_length':2,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
136
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
137
            }),
138
        ('annotated-knit-escape', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
139
            'cleanup':None,
140
            'factory':make_file_factory(True, HashEscapedPrefixMapper()),
141
            'graph':True,
142
            'key_length':2,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
143
            'support_partial_insertion': False,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
144
            }),
145
        ('plain-knit-pack', {
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
146
            'cleanup':cleanup_pack_knit,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
147
            'factory':make_pack_factory(True, True, 2),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
148
            'graph':True,
149
            'key_length':2,
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
150
            'support_partial_insertion': True,
3350.6.2 by Robert Collins
Prepare parameterised test environment.
151
            }),
152
        ]
153
    for test in iter_suite_tests(to_adapt):
154
        result.addTests(len_one_adapter.adapt(test))
155
        result.addTests(len_two_adapter.adapt(test))
156
    return result
157
158
3350.3.11 by Robert Collins
Test inserting a stream that overlaps the current content of a knit does not error.
159
def get_diamond_vf(f, trailing_eol=True, left_only=False):
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
160
    """Get a diamond graph to exercise deltas and merges.
161
    
162
    :param trailing_eol: If True end the last line with \n.
163
    """
164
    parents = {
165
        'origin': (),
166
        'base': (('origin',),),
167
        'left': (('base',),),
168
        'right': (('base',),),
169
        'merged': (('left',), ('right',)),
170
        }
171
    # insert a diamond graph to exercise deltas and merges.
172
    if trailing_eol:
173
        last_char = '\n'
174
    else:
175
        last_char = ''
176
    f.add_lines('origin', [], ['origin' + last_char])
177
    f.add_lines('base', ['origin'], ['base' + last_char])
178
    f.add_lines('left', ['base'], ['base\n', 'left' + last_char])
3350.3.11 by Robert Collins
Test inserting a stream that overlaps the current content of a knit does not error.
179
    if not left_only:
180
        f.add_lines('right', ['base'],
181
            ['base\n', 'right' + last_char])
182
        f.add_lines('merged', ['left', 'right'],
183
            ['base\n', 'left\n', 'right\n', 'merged' + last_char])
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
184
    return f, parents
185
186
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
187
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
188
    nograph=False, nokeys=False):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
189
    """Get a diamond graph to exercise deltas and merges.
190
191
    This creates a 5-node graph in files. If files supports 2-length keys two
192
    graphs are made to exercise the support for multiple ids.
193
    
194
    :param trailing_eol: If True end the last line with \n.
195
    :param key_length: The length of keys in files. Currently supports length 1
196
        and 2 keys.
197
    :param left_only: If True do not add the right and merged nodes.
198
    :param nograph: If True, do not provide parents to the add_lines calls;
199
        this is useful for tests that need inserted data but have graphless
200
        stores.
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
201
    :param nokeys: If True, pass None is as the key for all insertions.
202
        Currently implies nograph.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
203
    :return: The results of the add_lines calls.
204
    """
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
205
    if nokeys:
206
        nograph = True
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
207
    if key_length == 1:
208
        prefixes = [()]
209
    else:
210
        prefixes = [('FileA',), ('FileB',)]
211
    # insert a diamond graph to exercise deltas and merges.
212
    if trailing_eol:
213
        last_char = '\n'
214
    else:
215
        last_char = ''
216
    result = []
217
    def get_parents(suffix_list):
218
        if nograph:
219
            return ()
220
        else:
221
            result = [prefix + suffix for suffix in suffix_list]
222
            return result
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
223
    def get_key(suffix):
224
        if nokeys:
225
            return (None, )
226
        else:
227
            return (suffix,)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
228
    # we loop over each key because that spreads the inserts across prefixes,
229
    # which is how commit operates.
230
    for prefix in prefixes:
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
231
        result.append(files.add_lines(prefix + get_key('origin'), (),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
232
            ['origin' + last_char]))
233
    for prefix in prefixes:
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
234
        result.append(files.add_lines(prefix + get_key('base'),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
235
            get_parents([('origin',)]), ['base' + last_char]))
236
    for prefix in prefixes:
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
237
        result.append(files.add_lines(prefix + get_key('left'),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
238
            get_parents([('base',)]),
239
            ['base\n', 'left' + last_char]))
240
    if not left_only:
241
        for prefix in prefixes:
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
242
            result.append(files.add_lines(prefix + get_key('right'),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
243
                get_parents([('base',)]),
244
                ['base\n', 'right' + last_char]))
245
        for prefix in prefixes:
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
246
            result.append(files.add_lines(prefix + get_key('merged'),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
247
                get_parents([('left',), ('right',)]),
248
                ['base\n', 'left\n', 'right\n', 'merged' + last_char]))
249
    return result
250
251
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
252
class VersionedFileTestMixIn(object):
253
    """A mixin test class for testing VersionedFiles.
254
255
    This is not an adaptor-style test at this point because
256
    theres no dynamic substitution of versioned file implementations,
257
    they are strictly controlled by their owning repositories.
258
    """
259
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
260
    def get_transaction(self):
261
        if not hasattr(self, '_transaction'):
262
            self._transaction = None
263
        return self._transaction
264
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
265
    def test_add(self):
266
        f = self.get_file()
267
        f.add_lines('r0', [], ['a\n', 'b\n'])
268
        f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
269
        def verify_file(f):
270
            versions = f.versions()
271
            self.assertTrue('r0' in versions)
272
            self.assertTrue('r1' in versions)
273
            self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
274
            self.assertEquals(f.get_text('r0'), 'a\nb\n')
275
            self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
1563.2.18 by Robert Collins
get knit repositories really using knits for text storage.
276
            self.assertEqual(2, len(f))
277
            self.assertEqual(2, f.num_versions())
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
278
    
279
            self.assertRaises(RevisionNotPresent,
280
                f.add_lines, 'r2', ['foo'], [])
281
            self.assertRaises(RevisionAlreadyPresent,
282
                f.add_lines, 'r1', [], [])
283
        verify_file(f)
1666.1.6 by Robert Collins
Make knit the default format.
284
        # this checks that reopen with create=True does not break anything.
285
        f = self.reopen_file(create=True)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
286
        verify_file(f)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
287
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
288
    def test_adds_with_parent_texts(self):
289
        f = self.get_file()
290
        parent_texts = {}
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
291
        _, _, parent_texts['r0'] = f.add_lines('r0', [], ['a\n', 'b\n'])
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
292
        try:
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
293
            _, _, parent_texts['r1'] = f.add_lines_with_ghosts('r1',
294
                ['r0', 'ghost'], ['b\n', 'c\n'], parent_texts=parent_texts)
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
295
        except NotImplementedError:
296
            # if the format doesn't support ghosts, just add normally.
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
297
            _, _, parent_texts['r1'] = f.add_lines('r1',
298
                ['r0'], ['b\n', 'c\n'], parent_texts=parent_texts)
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
299
        f.add_lines('r2', ['r1'], ['c\n', 'd\n'], parent_texts=parent_texts)
300
        self.assertNotEqual(None, parent_texts['r0'])
301
        self.assertNotEqual(None, parent_texts['r1'])
302
        def verify_file(f):
303
            versions = f.versions()
304
            self.assertTrue('r0' in versions)
305
            self.assertTrue('r1' in versions)
306
            self.assertTrue('r2' in versions)
307
            self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
308
            self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
309
            self.assertEquals(f.get_lines('r2'), ['c\n', 'd\n'])
310
            self.assertEqual(3, f.num_versions())
311
            origins = f.annotate('r1')
312
            self.assertEquals(origins[0][0], 'r0')
313
            self.assertEquals(origins[1][0], 'r1')
314
            origins = f.annotate('r2')
315
            self.assertEquals(origins[0][0], 'r1')
316
            self.assertEquals(origins[1][0], 'r2')
317
318
        verify_file(f)
319
        f = self.reopen_file()
320
        verify_file(f)
321
2805.6.7 by Robert Collins
Review feedback.
322
    def test_add_unicode_content(self):
323
        # unicode content is not permitted in versioned files. 
324
        # versioned files version sequences of bytes only.
325
        vf = self.get_file()
326
        self.assertRaises(errors.BzrBadParameterUnicode,
327
            vf.add_lines, 'a', [], ['a\n', u'b\n', 'c\n'])
328
        self.assertRaises(
329
            (errors.BzrBadParameterUnicode, NotImplementedError),
330
            vf.add_lines_with_ghosts, 'a', [], ['a\n', u'b\n', 'c\n'])
331
2520.4.150 by Aaron Bentley
Test that non-Weave uses left_matching_blocks for add_lines
332
    def test_add_follows_left_matching_blocks(self):
333
        """If we change left_matching_blocks, delta changes
334
335
        Note: There are multiple correct deltas in this case, because
336
        we start with 1 "a" and we get 3.
337
        """
338
        vf = self.get_file()
339
        if isinstance(vf, WeaveFile):
340
            raise TestSkipped("WeaveFile ignores left_matching_blocks")
341
        vf.add_lines('1', [], ['a\n'])
342
        vf.add_lines('2', ['1'], ['a\n', 'a\n', 'a\n'],
343
                     left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
2794.1.2 by Robert Collins
Nuke versioned file add/get delta support, allowing easy simplification of unannotated Content, reducing memory copies and friction during commit on unannotated texts.
344
        self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('2'))
2520.4.150 by Aaron Bentley
Test that non-Weave uses left_matching_blocks for add_lines
345
        vf.add_lines('3', ['1'], ['a\n', 'a\n', 'a\n'],
346
                     left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
2794.1.2 by Robert Collins
Nuke versioned file add/get delta support, allowing easy simplification of unannotated Content, reducing memory copies and friction during commit on unannotated texts.
347
        self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('3'))
2520.4.150 by Aaron Bentley
Test that non-Weave uses left_matching_blocks for add_lines
348
2805.6.7 by Robert Collins
Review feedback.
349
    def test_inline_newline_throws(self):
350
        # \r characters are not permitted in lines being added
351
        vf = self.get_file()
352
        self.assertRaises(errors.BzrBadParameterContainsNewline, 
353
            vf.add_lines, 'a', [], ['a\n\n'])
354
        self.assertRaises(
355
            (errors.BzrBadParameterContainsNewline, NotImplementedError),
356
            vf.add_lines_with_ghosts, 'a', [], ['a\n\n'])
357
        # but inline CR's are allowed
358
        vf.add_lines('a', [], ['a\r\n'])
359
        try:
360
            vf.add_lines_with_ghosts('b', [], ['a\r\n'])
361
        except NotImplementedError:
362
            pass
363
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
364
    def test_add_reserved(self):
365
        vf = self.get_file()
366
        self.assertRaises(errors.ReservedId,
367
            vf.add_lines, 'a:', [], ['a\n', 'b\n', 'c\n'])
368
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
369
    def test_add_lines_nostoresha(self):
370
        """When nostore_sha is supplied using old content raises."""
371
        vf = self.get_file()
372
        empty_text = ('a', [])
373
        sample_text_nl = ('b', ["foo\n", "bar\n"])
374
        sample_text_no_nl = ('c', ["foo\n", "bar"])
375
        shas = []
376
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
377
            sha, _, _ = vf.add_lines(version, [], lines)
378
            shas.append(sha)
379
        # we now have a copy of all the lines in the vf.
380
        for sha, (version, lines) in zip(
381
            shas, (empty_text, sample_text_nl, sample_text_no_nl)):
382
            self.assertRaises(errors.ExistingContent,
383
                vf.add_lines, version + "2", [], lines,
384
                nostore_sha=sha)
385
            # and no new version should have been added.
386
            self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
387
                version + "2")
388
2803.1.1 by Robert Collins
Fix typo in ghosts version of test_add_lines_nostoresha.
389
    def test_add_lines_with_ghosts_nostoresha(self):
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
390
        """When nostore_sha is supplied using old content raises."""
391
        vf = self.get_file()
392
        empty_text = ('a', [])
393
        sample_text_nl = ('b', ["foo\n", "bar\n"])
394
        sample_text_no_nl = ('c', ["foo\n", "bar"])
395
        shas = []
396
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
397
            sha, _, _ = vf.add_lines(version, [], lines)
398
            shas.append(sha)
399
        # we now have a copy of all the lines in the vf.
400
        # is the test applicable to this vf implementation?
401
        try:
402
            vf.add_lines_with_ghosts('d', [], [])
403
        except NotImplementedError:
404
            raise TestSkipped("add_lines_with_ghosts is optional")
405
        for sha, (version, lines) in zip(
406
            shas, (empty_text, sample_text_nl, sample_text_no_nl)):
407
            self.assertRaises(errors.ExistingContent,
408
                vf.add_lines_with_ghosts, version + "2", [], lines,
409
                nostore_sha=sha)
410
            # and no new version should have been added.
411
            self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
412
                version + "2")
413
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
414
    def test_add_lines_return_value(self):
415
        # add_lines should return the sha1 and the text size.
416
        vf = self.get_file()
417
        empty_text = ('a', [])
418
        sample_text_nl = ('b', ["foo\n", "bar\n"])
419
        sample_text_no_nl = ('c', ["foo\n", "bar"])
420
        # check results for the three cases:
421
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
422
            # the first two elements are the same for all versioned files:
423
            # - the digest and the size of the text. For some versioned files
424
            #   additional data is returned in additional tuple elements.
425
            result = vf.add_lines(version, [], lines)
426
            self.assertEqual(3, len(result))
427
            self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
428
                result[0:2])
429
        # parents should not affect the result:
430
        lines = sample_text_nl[1]
431
        self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
432
            vf.add_lines('d', ['b', 'c'], lines)[0:2])
433
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
434
    def test_get_reserved(self):
435
        vf = self.get_file()
436
        self.assertRaises(errors.ReservedId, vf.get_texts, ['b:'])
437
        self.assertRaises(errors.ReservedId, vf.get_lines, 'b:')
438
        self.assertRaises(errors.ReservedId, vf.get_text, 'b:')
439
3468.2.4 by Martin Pool
Test and fix #234748 problems in trailing newline diffs
440
    def test_add_unchanged_last_line_noeol_snapshot(self):
441
        """Add a text with an unchanged last line with no eol should work."""
442
        # Test adding this in a number of chain lengths; because the interface
443
        # for VersionedFile does not allow forcing a specific chain length, we
444
        # just use a small base to get the first snapshot, then a much longer
445
        # first line for the next add (which will make the third add snapshot)
446
        # and so on. 20 has been chosen as an aribtrary figure - knits use 200
447
        # as a capped delta length, but ideally we would have some way of
448
        # tuning the test to the store (e.g. keep going until a snapshot
449
        # happens).
450
        for length in range(20):
451
            version_lines = {}
452
            vf = self.get_file('case-%d' % length)
453
            prefix = 'step-%d'
454
            parents = []
455
            for step in range(length):
456
                version = prefix % step
457
                lines = (['prelude \n'] * step) + ['line']
458
                vf.add_lines(version, parents, lines)
459
                version_lines[version] = lines
460
                parents = [version]
461
            vf.add_lines('no-eol', parents, ['line'])
462
            vf.get_texts(version_lines.keys())
463
            self.assertEqualDiff('line', vf.get_text('no-eol'))
464
465
    def test_get_texts_eol_variation(self):
466
        # similar to the failure in <http://bugs.launchpad.net/234748>
467
        vf = self.get_file()
468
        sample_text_nl = ["line\n"]
469
        sample_text_no_nl = ["line"]
470
        versions = []
471
        version_lines = {}
472
        parents = []
473
        for i in range(4):
474
            version = 'v%d' % i
475
            if i % 2:
476
                lines = sample_text_nl
477
            else:
478
                lines = sample_text_no_nl
479
            # left_matching blocks is an internal api; it operates on the
480
            # *internal* representation for a knit, which is with *all* lines
481
            # being normalised to end with \n - even the final line in a no_nl
482
            # file. Using it here ensures that a broken internal implementation
483
            # (which is what this test tests) will generate a correct line
484
            # delta (which is to say, an empty delta).
485
            vf.add_lines(version, parents, lines,
486
                left_matching_blocks=[(0, 0, 1)])
487
            parents = [version]
488
            versions.append(version)
489
            version_lines[version] = lines
490
        vf.check()
491
        vf.get_texts(versions)
492
        vf.get_texts(reversed(versions))
493
3460.2.1 by Robert Collins
* Inserting a bundle which changes the contents of a file with no trailing
494
    def test_add_lines_with_matching_blocks_noeol_last_line(self):
495
        """Add a text with an unchanged last line with no eol should work."""
496
        from bzrlib import multiparent
497
        # Hand verified sha1 of the text we're adding.
498
        sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
499
        # Create a mpdiff which adds a new line before the trailing line, and
500
        # reuse the last line unaltered (which can cause annotation reuse).
501
        # Test adding this in two situations:
502
        # On top of a new insertion
503
        vf = self.get_file('fulltext')
504
        vf.add_lines('noeol', [], ['line'])
505
        vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
506
            left_matching_blocks=[(0, 1, 1)])
507
        self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
508
        # On top of a delta
509
        vf = self.get_file('delta')
510
        vf.add_lines('base', [], ['line'])
511
        vf.add_lines('noeol', ['base'], ['prelude\n', 'line'])
512
        vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
513
            left_matching_blocks=[(1, 1, 1)])
514
        self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
515
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
516
    def test_make_mpdiffs(self):
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
517
        from bzrlib import multiparent
518
        vf = self.get_file('foo')
519
        sha1s = self._setup_for_deltas(vf)
520
        new_vf = self.get_file('bar')
521
        for version in multiparent.topo_iter(vf):
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
522
            mpdiff = vf.make_mpdiffs([version])[0]
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
523
            new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
524
                                 vf.get_sha1s([version])[version], mpdiff)])
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
525
            self.assertEqualDiff(vf.get_text(version),
526
                                 new_vf.get_text(version))
527
3453.3.2 by John Arbash Meinel
Add a test case for the first loop, unable to find a way to trigger the second loop
528
    def test_make_mpdiffs_with_ghosts(self):
529
        vf = self.get_file('foo')
3453.3.4 by John Arbash Meinel
Skip the new test for old weave formats that don't support ghosts
530
        try:
531
            vf.add_lines_with_ghosts('text', ['ghost'], ['line\n'])
532
        except NotImplementedError:
533
            # old Weave formats do not allow ghosts
534
            return
3453.3.2 by John Arbash Meinel
Add a test case for the first loop, unable to find a way to trigger the second loop
535
        self.assertRaises(errors.RevisionNotPresent, vf.make_mpdiffs, ['ghost'])
536
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
537
    def _setup_for_deltas(self, f):
2794.1.2 by Robert Collins
Nuke versioned file add/get delta support, allowing easy simplification of unannotated Content, reducing memory copies and friction during commit on unannotated texts.
538
        self.assertFalse(f.has_version('base'))
1596.2.36 by Robert Collins
add a get_delta api to versioned_file.
539
        # add texts that should trip the knit maximum delta chain threshold
540
        # as well as doing parallel chains of data in knits.
541
        # this is done by two chains of 25 insertions
542
        f.add_lines('base', [], ['line\n'])
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
543
        f.add_lines('noeol', ['base'], ['line'])
544
        # detailed eol tests:
545
        # shared last line with parent no-eol
546
        f.add_lines('noeolsecond', ['noeol'], ['line\n', 'line'])
547
        # differing last line with parent, both no-eol
548
        f.add_lines('noeolnotshared', ['noeolsecond'], ['line\n', 'phone'])
549
        # add eol following a noneol parent, change content
550
        f.add_lines('eol', ['noeol'], ['phone\n'])
551
        # add eol following a noneol parent, no change content
552
        f.add_lines('eolline', ['noeol'], ['line\n'])
553
        # noeol with no parents:
554
        f.add_lines('noeolbase', [], ['line'])
555
        # noeol preceeding its leftmost parent in the output:
556
        # this is done by making it a merge of two parents with no common
557
        # anestry: noeolbase and noeol with the 
558
        # later-inserted parent the leftmost.
559
        f.add_lines('eolbeforefirstparent', ['noeolbase', 'noeol'], ['line'])
560
        # two identical eol texts
561
        f.add_lines('noeoldup', ['noeol'], ['line'])
1596.2.36 by Robert Collins
add a get_delta api to versioned_file.
562
        next_parent = 'base'
563
        text_name = 'chain1-'
564
        text = ['line\n']
565
        sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
566
                 1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
567
                 2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
568
                 3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
569
                 4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
570
                 5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
571
                 6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
572
                 7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
573
                 8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
574
                 9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
575
                 10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
576
                 11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
577
                 12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
578
                 13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
579
                 14:'2c4b1736566b8ca6051e668de68650686a3922f2',
580
                 15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
581
                 16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
582
                 17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
583
                 18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
584
                 19:'1ebed371807ba5935958ad0884595126e8c4e823',
585
                 20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
586
                 21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
587
                 22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
588
                 23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
589
                 24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
590
                 25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
591
                 }
592
        for depth in range(26):
593
            new_version = text_name + '%s' % depth
594
            text = text + ['line\n']
595
            f.add_lines(new_version, [next_parent], text)
596
            next_parent = new_version
597
        next_parent = 'base'
598
        text_name = 'chain2-'
599
        text = ['line\n']
600
        for depth in range(26):
601
            new_version = text_name + '%s' % depth
602
            text = text + ['line\n']
603
            f.add_lines(new_version, [next_parent], text)
604
            next_parent = new_version
1596.2.38 by Robert Collins
rollback from using deltas to using fulltexts - deltas need more work to be ready.
605
        return sha1s
1596.2.37 by Robert Collins
Switch to delta based content copying in the generic versioned file copier.
606
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
607
    def test_ancestry(self):
608
        f = self.get_file()
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
609
        self.assertEqual([], f.get_ancestry([]))
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
610
        f.add_lines('r0', [], ['a\n', 'b\n'])
611
        f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
612
        f.add_lines('r2', ['r0'], ['b\n', 'c\n'])
613
        f.add_lines('r3', ['r2'], ['b\n', 'c\n'])
614
        f.add_lines('rM', ['r1', 'r2'], ['b\n', 'c\n'])
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
615
        self.assertEqual([], f.get_ancestry([]))
1563.2.35 by Robert Collins
cleanup deprecation warnings and finish conversion so the inventory is knit based too.
616
        versions = f.get_ancestry(['rM'])
617
        # there are some possibilities:
618
        # r0 r1 r2 rM r3
619
        # r0 r1 r2 r3 rM
620
        # etc
621
        # so we check indexes
622
        r0 = versions.index('r0')
623
        r1 = versions.index('r1')
624
        r2 = versions.index('r2')
625
        self.assertFalse('r3' in versions)
626
        rM = versions.index('rM')
627
        self.assertTrue(r0 < r1)
628
        self.assertTrue(r0 < r2)
629
        self.assertTrue(r1 < rM)
630
        self.assertTrue(r2 < rM)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
631
632
        self.assertRaises(RevisionNotPresent,
633
            f.get_ancestry, ['rM', 'rX'])
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
634
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
635
        self.assertEqual(set(f.get_ancestry('rM')),
636
            set(f.get_ancestry('rM', topo_sorted=False)))
637
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
638
    def test_mutate_after_finish(self):
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
639
        self._transaction = 'before'
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
640
        f = self.get_file()
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
641
        self._transaction = 'after'
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
642
        self.assertRaises(errors.OutSideTransaction, f.add_lines, '', [], [])
643
        self.assertRaises(errors.OutSideTransaction, f.add_lines_with_ghosts, '', [], [])
1563.2.7 by Robert Collins
add versioned file clear_cache entry.
644
        
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
645
    def test_copy_to(self):
646
        f = self.get_file()
647
        f.add_lines('0', [], ['a\n'])
648
        t = MemoryTransport()
649
        f.copy_to('foo', t)
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
650
        for suffix in self.get_factory().get_suffixes():
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
651
            self.assertTrue(t.has('foo' + suffix))
652
653
    def test_get_suffixes(self):
654
        f = self.get_file()
655
        # and should be a list
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
656
        self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
657
3287.5.1 by Robert Collins
Add VersionedFile.get_parent_map.
658
    def test_get_parent_map(self):
659
        f = self.get_file()
660
        f.add_lines('r0', [], ['a\n', 'b\n'])
661
        self.assertEqual(
662
            {'r0':()}, f.get_parent_map(['r0']))
663
        f.add_lines('r1', ['r0'], ['a\n', 'b\n'])
664
        self.assertEqual(
665
            {'r1':('r0',)}, f.get_parent_map(['r1']))
666
        self.assertEqual(
667
            {'r0':(),
668
             'r1':('r0',)},
669
            f.get_parent_map(['r0', 'r1']))
670
        f.add_lines('r2', [], ['a\n', 'b\n'])
671
        f.add_lines('r3', [], ['a\n', 'b\n'])
672
        f.add_lines('m', ['r0', 'r1', 'r2', 'r3'], ['a\n', 'b\n'])
673
        self.assertEqual(
674
            {'m':('r0', 'r1', 'r2', 'r3')}, f.get_parent_map(['m']))
675
        self.assertEqual({}, f.get_parent_map('y'))
676
        self.assertEqual(
677
            {'r0':(),
678
             'r1':('r0',)},
679
            f.get_parent_map(['r0', 'y', 'r1']))
680
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
681
    def test_annotate(self):
682
        f = self.get_file()
683
        f.add_lines('r0', [], ['a\n', 'b\n'])
684
        f.add_lines('r1', ['r0'], ['c\n', 'b\n'])
685
        origins = f.annotate('r1')
686
        self.assertEquals(origins[0][0], 'r1')
687
        self.assertEquals(origins[1][0], 'r0')
688
689
        self.assertRaises(RevisionNotPresent,
690
            f.annotate, 'foo')
691
1563.2.6 by Robert Collins
Start check tests for knits (pending), and remove dead code.
692
    def test_detection(self):
693
        # Test weaves detect corruption.
694
        #
695
        # Weaves contain a checksum of their texts.
696
        # When a text is extracted, this checksum should be
697
        # verified.
698
699
        w = self.get_file_corrupted_text()
700
701
        self.assertEqual('hello\n', w.get_text('v1'))
702
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
703
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
704
        self.assertRaises(errors.WeaveInvalidChecksum, w.check)
705
706
        w = self.get_file_corrupted_checksum()
707
708
        self.assertEqual('hello\n', w.get_text('v1'))
709
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
710
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
711
        self.assertRaises(errors.WeaveInvalidChecksum, w.check)
712
713
    def get_file_corrupted_text(self):
714
        """Return a versioned file with corrupt text but valid metadata."""
715
        raise NotImplementedError(self.get_file_corrupted_text)
716
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
717
    def reopen_file(self, name='foo'):
718
        """Open the versioned file from disk again."""
719
        raise NotImplementedError(self.reopen_file)
720
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
721
    def test_iter_lines_added_or_present_in_versions(self):
722
        # test that we get at least an equalset of the lines added by
723
        # versions in the weave 
724
        # the ordering here is to make a tree so that dumb searches have
725
        # more changes to muck up.
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
726
727
        class InstrumentedProgress(progress.DummyProgress):
728
729
            def __init__(self):
730
731
                progress.DummyProgress.__init__(self)
732
                self.updates = []
733
734
            def update(self, msg=None, current=None, total=None):
735
                self.updates.append((msg, current, total))
736
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
737
        vf = self.get_file()
738
        # add a base to get included
739
        vf.add_lines('base', [], ['base\n'])
740
        # add a ancestor to be included on one side
741
        vf.add_lines('lancestor', [], ['lancestor\n'])
742
        # add a ancestor to be included on the other side
743
        vf.add_lines('rancestor', ['base'], ['rancestor\n'])
744
        # add a child of rancestor with no eofile-nl
745
        vf.add_lines('child', ['rancestor'], ['base\n', 'child\n'])
746
        # add a child of lancestor and base to join the two roots
747
        vf.add_lines('otherchild',
748
                     ['lancestor', 'base'],
749
                     ['base\n', 'lancestor\n', 'otherchild\n'])
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
750
        def iter_with_versions(versions, expected):
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
751
            # now we need to see what lines are returned, and how often.
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
752
            lines = {}
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
753
            progress = InstrumentedProgress()
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
754
            # iterate over the lines
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
755
            for line in vf.iter_lines_added_or_present_in_versions(versions,
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
756
                pb=progress):
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
757
                lines.setdefault(line, 0)
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
758
                lines[line] += 1
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
759
            if []!= progress.updates:
2039.1.2 by Aaron Bentley
Tweak test to avoid catching assert
760
                self.assertEqual(expected, progress.updates)
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
761
            return lines
2147.1.3 by John Arbash Meinel
In knit.py we were re-using a variable in 2 loops, causing bogus progress messages to be generated.
762
        lines = iter_with_versions(['child', 'otherchild'],
763
                                   [('Walking content.', 0, 2),
764
                                    ('Walking content.', 1, 2),
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
765
                                    ('Walking content.', 2, 2)])
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
766
        # we must see child and otherchild
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
767
        self.assertTrue(lines[('child\n', 'child')] > 0)
768
        self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
769
        # we dont care if we got more than that.
770
        
771
        # test all lines
2147.1.3 by John Arbash Meinel
In knit.py we were re-using a variable in 2 loops, causing bogus progress messages to be generated.
772
        lines = iter_with_versions(None, [('Walking content.', 0, 5),
773
                                          ('Walking content.', 1, 5),
774
                                          ('Walking content.', 2, 5),
775
                                          ('Walking content.', 3, 5),
776
                                          ('Walking content.', 4, 5),
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
777
                                          ('Walking content.', 5, 5)])
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
778
        # all lines must be seen at least once
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
779
        self.assertTrue(lines[('base\n', 'base')] > 0)
780
        self.assertTrue(lines[('lancestor\n', 'lancestor')] > 0)
781
        self.assertTrue(lines[('rancestor\n', 'rancestor')] > 0)
782
        self.assertTrue(lines[('child\n', 'child')] > 0)
783
        self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
784
1594.2.8 by Robert Collins
add ghost aware apis to knits.
785
    def test_add_lines_with_ghosts(self):
786
        # some versioned file formats allow lines to be added with parent
787
        # information that is > than that in the format. Formats that do
788
        # not support this need to raise NotImplementedError on the
789
        # add_lines_with_ghosts api.
790
        vf = self.get_file()
791
        # add a revision with ghost parents
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
792
        # The preferred form is utf8, but we should translate when needed
793
        parent_id_unicode = u'b\xbfse'
794
        parent_id_utf8 = parent_id_unicode.encode('utf8')
1594.2.8 by Robert Collins
add ghost aware apis to knits.
795
        try:
2309.4.7 by John Arbash Meinel
Update VersionedFile tests to ensure that they can take Unicode,
796
            vf.add_lines_with_ghosts('notbxbfse', [parent_id_utf8], [])
1594.2.8 by Robert Collins
add ghost aware apis to knits.
797
        except NotImplementedError:
798
            # check the other ghost apis are also not implemented
799
            self.assertRaises(NotImplementedError, vf.get_ancestry_with_ghosts, ['foo'])
800
            self.assertRaises(NotImplementedError, vf.get_parents_with_ghosts, 'foo')
801
            return
2150.2.1 by Robert Collins
Correctly decode utf8 revision ids from knits when parsing, fixes a regression where a unicode revision id is stored correctly, but then indexed by the utf8 value on the next invocation of bzr, rather than the unicode value.
802
        vf = self.reopen_file()
1594.2.8 by Robert Collins
add ghost aware apis to knits.
803
        # test key graph related apis: getncestry, _graph, get_parents
804
        # has_version
805
        # - these are ghost unaware and must not be reflect ghosts
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
806
        self.assertEqual(['notbxbfse'], vf.get_ancestry('notbxbfse'))
807
        self.assertFalse(vf.has_version(parent_id_utf8))
1594.2.8 by Robert Collins
add ghost aware apis to knits.
808
        # we have _with_ghost apis to give us ghost information.
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
809
        self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry_with_ghosts(['notbxbfse']))
810
        self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
1594.2.8 by Robert Collins
add ghost aware apis to knits.
811
        # if we add something that is a ghost of another, it should correct the
812
        # results of the prior apis
2858.2.1 by Martin Pool
Remove most calls to safe_file_id and safe_revision_id.
813
        vf.add_lines(parent_id_utf8, [], [])
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
814
        self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry(['notbxbfse']))
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
815
        self.assertEqual({'notbxbfse':(parent_id_utf8,)},
816
            vf.get_parent_map(['notbxbfse']))
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
817
        self.assertTrue(vf.has_version(parent_id_utf8))
1594.2.8 by Robert Collins
add ghost aware apis to knits.
818
        # we have _with_ghost apis to give us ghost information.
2858.2.1 by Martin Pool
Remove most calls to safe_file_id and safe_revision_id.
819
        self.assertEqual([parent_id_utf8, 'notbxbfse'],
820
            vf.get_ancestry_with_ghosts(['notbxbfse']))
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
821
        self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
1594.2.8 by Robert Collins
add ghost aware apis to knits.
822
1594.2.9 by Robert Collins
Teach Knit repositories how to handle ghosts without corrupting at all.
823
    def test_add_lines_with_ghosts_after_normal_revs(self):
824
        # some versioned file formats allow lines to be added with parent
825
        # information that is > than that in the format. Formats that do
826
        # not support this need to raise NotImplementedError on the
827
        # add_lines_with_ghosts api.
828
        vf = self.get_file()
829
        # probe for ghost support
830
        try:
3287.6.5 by Robert Collins
Deprecate VersionedFile.has_ghost.
831
            vf.add_lines_with_ghosts('base', [], ['line\n', 'line_b\n'])
1594.2.9 by Robert Collins
Teach Knit repositories how to handle ghosts without corrupting at all.
832
        except NotImplementedError:
833
            return
834
        vf.add_lines_with_ghosts('references_ghost',
835
                                 ['base', 'a_ghost'],
836
                                 ['line\n', 'line_b\n', 'line_c\n'])
837
        origins = vf.annotate('references_ghost')
838
        self.assertEquals(('base', 'line\n'), origins[0])
839
        self.assertEquals(('base', 'line_b\n'), origins[1])
840
        self.assertEquals(('references_ghost', 'line_c\n'), origins[2])
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
841
842
    def test_readonly_mode(self):
843
        transport = get_transport(self.get_url('.'))
844
        factory = self.get_factory()
845
        vf = factory('id', transport, 0777, create=True, access_mode='w')
846
        vf = factory('id', transport, access_mode='r')
847
        self.assertRaises(errors.ReadOnlyError, vf.add_lines, 'base', [], [])
848
        self.assertRaises(errors.ReadOnlyError,
849
                          vf.add_lines_with_ghosts,
850
                          'base',
851
                          [],
852
                          [])
1666.1.6 by Robert Collins
Make knit the default format.
853
    
3316.2.9 by Robert Collins
* ``VersionedFile.get_sha1`` is deprecated, please use
854
    def test_get_sha1s(self):
1666.1.6 by Robert Collins
Make knit the default format.
855
        # check the sha1 data is available
856
        vf = self.get_file()
857
        # a simple file
858
        vf.add_lines('a', [], ['a\n'])
859
        # the same file, different metadata
860
        vf.add_lines('b', ['a'], ['a\n'])
861
        # a file differing only in last newline.
862
        vf.add_lines('c', [], ['a'])
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
863
        self.assertEqual({
864
            'a': '3f786850e387550fdab836ed7e6dc881de23001b',
865
            'c': '86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
866
            'b': '3f786850e387550fdab836ed7e6dc881de23001b',
867
            },
868
            vf.get_sha1s(['a', 'c', 'b']))
1594.2.9 by Robert Collins
Teach Knit repositories how to handle ghosts without corrupting at all.
869
        
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
870
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
871
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
872
873
    def get_file(self, name='foo'):
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
874
        return WeaveFile(name, get_transport(self.get_url('.')), create=True,
875
            get_scope=self.get_transaction)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
876
1563.2.6 by Robert Collins
Start check tests for knits (pending), and remove dead code.
877
    def get_file_corrupted_text(self):
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
878
        w = WeaveFile('foo', get_transport(self.get_url('.')), create=True,
879
            get_scope=self.get_transaction)
1563.2.13 by Robert Collins
InterVersionedFile implemented.
880
        w.add_lines('v1', [], ['hello\n'])
881
        w.add_lines('v2', ['v1'], ['hello\n', 'there\n'])
1563.2.6 by Robert Collins
Start check tests for knits (pending), and remove dead code.
882
        
883
        # We are going to invasively corrupt the text
884
        # Make sure the internals of weave are the same
885
        self.assertEqual([('{', 0)
886
                        , 'hello\n'
887
                        , ('}', None)
888
                        , ('{', 1)
889
                        , 'there\n'
890
                        , ('}', None)
891
                        ], w._weave)
892
        
893
        self.assertEqual(['f572d396fae9206628714fb2ce00f72e94f2258f'
894
                        , '90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
895
                        ], w._sha1s)
896
        w.check()
897
        
898
        # Corrupted
899
        w._weave[4] = 'There\n'
900
        return w
901
902
    def get_file_corrupted_checksum(self):
903
        w = self.get_file_corrupted_text()
904
        # Corrected
905
        w._weave[4] = 'there\n'
906
        self.assertEqual('hello\nthere\n', w.get_text('v2'))
907
        
908
        #Invalid checksum, first digit changed
909
        w._sha1s[1] =  'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
910
        return w
911
1666.1.6 by Robert Collins
Make knit the default format.
912
    def reopen_file(self, name='foo', create=False):
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
913
        return WeaveFile(name, get_transport(self.get_url('.')), create=create,
914
            get_scope=self.get_transaction)
1563.2.9 by Robert Collins
Update versionedfile api tests to ensure that data is available after every operation.
915
1563.2.25 by Robert Collins
Merge in upstream.
916
    def test_no_implicit_create(self):
917
        self.assertRaises(errors.NoSuchFile,
918
                          WeaveFile,
919
                          'foo',
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
920
                          get_transport(self.get_url('.')),
921
                          get_scope=self.get_transaction)
1563.2.25 by Robert Collins
Merge in upstream.
922
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
923
    def get_factory(self):
924
        return WeaveFile
925
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
926
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
927
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
928
929
    def setUp(self):
930
        TestCaseWithMemoryTransport.setUp(self)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
931
        mapper = PrefixMapper()
932
        factory = make_file_factory(True, mapper)
933
        self.vf1 = factory(self.get_transport('root-1'))
934
        self.vf2 = factory(self.get_transport('root-2'))
935
        self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
936
        self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
937
938
    def test_add_lines(self):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
939
        self.plan_merge_vf.add_lines(('root', 'a:'), [], [])
940
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
941
            ('root', 'a'), [], [])
942
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
943
            ('root', 'a:'), None, [])
944
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
945
            ('root', 'a:'), [], None)
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
946
947
    def setup_abcde(self):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
948
        self.vf1.add_lines(('root', 'A'), [], ['a'])
949
        self.vf1.add_lines(('root', 'B'), [('root', 'A')], ['b'])
950
        self.vf2.add_lines(('root', 'C'), [], ['c'])
951
        self.vf2.add_lines(('root', 'D'), [('root', 'C')], ['d'])
952
        self.plan_merge_vf.add_lines(('root', 'E:'),
953
            [('root', 'B'), ('root', 'D')], ['e'])
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
954
955
    def test_get_parents(self):
956
        self.setup_abcde()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
957
        self.assertEqual({('root', 'B'):(('root', 'A'),)},
958
            self.plan_merge_vf.get_parent_map([('root', 'B')]))
959
        self.assertEqual({('root', 'D'):(('root', 'C'),)},
960
            self.plan_merge_vf.get_parent_map([('root', 'D')]))
961
        self.assertEqual({('root', 'E:'):(('root', 'B'),('root', 'D'))},
962
            self.plan_merge_vf.get_parent_map([('root', 'E:')]))
963
        self.assertEqual({},
964
            self.plan_merge_vf.get_parent_map([('root', 'F')]))
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
965
        self.assertEqual({
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
966
                ('root', 'B'):(('root', 'A'),),
967
                ('root', 'D'):(('root', 'C'),),
968
                ('root', 'E:'):(('root', 'B'),('root', 'D')),
969
                },
970
            self.plan_merge_vf.get_parent_map(
971
                [('root', 'B'), ('root', 'D'), ('root', 'E:'), ('root', 'F')]))
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
972
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
973
    def test_get_record_stream(self):
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
974
        self.setup_abcde()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
975
        def get_record(suffix):
976
            return self.plan_merge_vf.get_record_stream(
977
                [('root', suffix)], 'unordered', True).next()
978
        self.assertEqual('a', get_record('A').get_bytes_as('fulltext'))
979
        self.assertEqual('c', get_record('C').get_bytes_as('fulltext'))
980
        self.assertEqual('e', get_record('E:').get_bytes_as('fulltext'))
981
        self.assertEqual('absent', get_record('F').storage_kind)
1666.1.1 by Robert Collins
Add trivial http-using test for versioned files.
982
983
984
class TestReadonlyHttpMixin(object):
985
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
986
    def get_transaction(self):
987
        return 1
988
1666.1.1 by Robert Collins
Add trivial http-using test for versioned files.
989
    def test_readonly_http_works(self):
990
        # we should be able to read from http with a versioned file.
991
        vf = self.get_file()
1666.1.6 by Robert Collins
Make knit the default format.
992
        # try an empty file access
993
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
994
        self.assertEqual([], readonly_vf.versions())
995
        # now with feeling.
1666.1.1 by Robert Collins
Add trivial http-using test for versioned files.
996
        vf.add_lines('1', [], ['a\n'])
997
        vf.add_lines('2', ['1'], ['b\n', 'a\n'])
998
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
1666.1.6 by Robert Collins
Make knit the default format.
999
        self.assertEqual(['1', '2'], vf.versions())
1666.1.1 by Robert Collins
Add trivial http-using test for versioned files.
1000
        for version in readonly_vf.versions():
1001
            readonly_vf.get_lines(version)
1002
1003
1004
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
1005
1006
    def get_file(self):
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
1007
        return WeaveFile('foo', get_transport(self.get_url('.')), create=True,
1008
            get_scope=self.get_transaction)
1666.1.1 by Robert Collins
Add trivial http-using test for versioned files.
1009
1010
    def get_factory(self):
1011
        return WeaveFile
1012
1013
1664.2.9 by Aaron Bentley
Ported weave merge test to versionedfile
1014
class MergeCasesMixin(object):
1015
1016
    def doMerge(self, base, a, b, mp):
1017
        from cStringIO import StringIO
1018
        from textwrap import dedent
1019
1020
        def addcrlf(x):
1021
            return x + '\n'
1022
        
1023
        w = self.get_file()
1024
        w.add_lines('text0', [], map(addcrlf, base))
1025
        w.add_lines('text1', ['text0'], map(addcrlf, a))
1026
        w.add_lines('text2', ['text0'], map(addcrlf, b))
1027
1028
        self.log_contents(w)
1029
1030
        self.log('merge plan:')
1031
        p = list(w.plan_merge('text1', 'text2'))
1032
        for state, line in p:
1033
            if line:
1034
                self.log('%12s | %s' % (state, line[:-1]))
1035
1036
        self.log('merge:')
1037
        mt = StringIO()
1038
        mt.writelines(w.weave_merge(p))
1039
        mt.seek(0)
1040
        self.log(mt.getvalue())
1041
1042
        mp = map(addcrlf, mp)
1043
        self.assertEqual(mt.readlines(), mp)
1044
        
1045
        
1046
    def testOneInsert(self):
1047
        self.doMerge([],
1048
                     ['aa'],
1049
                     [],
1050
                     ['aa'])
1051
1052
    def testSeparateInserts(self):
1053
        self.doMerge(['aaa', 'bbb', 'ccc'],
1054
                     ['aaa', 'xxx', 'bbb', 'ccc'],
1055
                     ['aaa', 'bbb', 'yyy', 'ccc'],
1056
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
1057
1058
    def testSameInsert(self):
1059
        self.doMerge(['aaa', 'bbb', 'ccc'],
1060
                     ['aaa', 'xxx', 'bbb', 'ccc'],
1061
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'],
1062
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
1063
    overlappedInsertExpected = ['aaa', 'xxx', 'yyy', 'bbb']
1064
    def testOverlappedInsert(self):
1065
        self.doMerge(['aaa', 'bbb'],
1066
                     ['aaa', 'xxx', 'yyy', 'bbb'],
1067
                     ['aaa', 'xxx', 'bbb'], self.overlappedInsertExpected)
1068
1069
        # really it ought to reduce this to 
1070
        # ['aaa', 'xxx', 'yyy', 'bbb']
1071
1072
1073
    def testClashReplace(self):
1074
        self.doMerge(['aaa'],
1075
                     ['xxx'],
1076
                     ['yyy', 'zzz'],
1077
                     ['<<<<<<< ', 'xxx', '=======', 'yyy', 'zzz', 
1078
                      '>>>>>>> '])
1079
1080
    def testNonClashInsert1(self):
1081
        self.doMerge(['aaa'],
1082
                     ['xxx', 'aaa'],
1083
                     ['yyy', 'zzz'],
1084
                     ['<<<<<<< ', 'xxx', 'aaa', '=======', 'yyy', 'zzz', 
1085
                      '>>>>>>> '])
1086
1087
    def testNonClashInsert2(self):
1088
        self.doMerge(['aaa'],
1089
                     ['aaa'],
1090
                     ['yyy', 'zzz'],
1091
                     ['yyy', 'zzz'])
1092
1093
1094
    def testDeleteAndModify(self):
1095
        """Clashing delete and modification.
1096
1097
        If one side modifies a region and the other deletes it then
1098
        there should be a conflict with one side blank.
1099
        """
1100
1101
        #######################################
1102
        # skippd, not working yet
1103
        return
1104
        
1105
        self.doMerge(['aaa', 'bbb', 'ccc'],
1106
                     ['aaa', 'ddd', 'ccc'],
1107
                     ['aaa', 'ccc'],
1108
                     ['<<<<<<<< ', 'aaa', '=======', '>>>>>>> ', 'ccc'])
1109
1110
    def _test_merge_from_strings(self, base, a, b, expected):
1111
        w = self.get_file()
1112
        w.add_lines('text0', [], base.splitlines(True))
1113
        w.add_lines('text1', ['text0'], a.splitlines(True))
1114
        w.add_lines('text2', ['text0'], b.splitlines(True))
1115
        self.log('merge plan:')
1116
        p = list(w.plan_merge('text1', 'text2'))
1117
        for state, line in p:
1118
            if line:
1119
                self.log('%12s | %s' % (state, line[:-1]))
1120
        self.log('merge result:')
1121
        result_text = ''.join(w.weave_merge(p))
1122
        self.log(result_text)
1123
        self.assertEqualDiff(result_text, expected)
1124
1125
    def test_weave_merge_conflicts(self):
1126
        # does weave merge properly handle plans that end with unchanged?
1127
        result = ''.join(self.get_file().weave_merge([('new-a', 'hello\n')]))
1128
        self.assertEqual(result, 'hello\n')
1129
1130
    def test_deletion_extended(self):
1131
        """One side deletes, the other deletes more.
1132
        """
1133
        base = """\
1134
            line 1
1135
            line 2
1136
            line 3
1137
            """
1138
        a = """\
1139
            line 1
1140
            line 2
1141
            """
1142
        b = """\
1143
            line 1
1144
            """
1145
        result = """\
1146
            line 1
1147
            """
1148
        self._test_merge_from_strings(base, a, b, result)
1149
1150
    def test_deletion_overlap(self):
1151
        """Delete overlapping regions with no other conflict.
1152
1153
        Arguably it'd be better to treat these as agreement, rather than 
1154
        conflict, but for now conflict is safer.
1155
        """
1156
        base = """\
1157
            start context
1158
            int a() {}
1159
            int b() {}
1160
            int c() {}
1161
            end context
1162
            """
1163
        a = """\
1164
            start context
1165
            int a() {}
1166
            end context
1167
            """
1168
        b = """\
1169
            start context
1170
            int c() {}
1171
            end context
1172
            """
1173
        result = """\
1174
            start context
1175
<<<<<<< 
1176
            int a() {}
1177
=======
1178
            int c() {}
1179
>>>>>>> 
1180
            end context
1181
            """
1182
        self._test_merge_from_strings(base, a, b, result)
1183
1184
    def test_agreement_deletion(self):
1185
        """Agree to delete some lines, without conflicts."""
1186
        base = """\
1187
            start context
1188
            base line 1
1189
            base line 2
1190
            end context
1191
            """
1192
        a = """\
1193
            start context
1194
            base line 1
1195
            end context
1196
            """
1197
        b = """\
1198
            start context
1199
            base line 1
1200
            end context
1201
            """
1202
        result = """\
1203
            start context
1204
            base line 1
1205
            end context
1206
            """
1207
        self._test_merge_from_strings(base, a, b, result)
1208
1209
    def test_sync_on_deletion(self):
1210
        """Specific case of merge where we can synchronize incorrectly.
1211
        
1212
        A previous version of the weave merge concluded that the two versions
1213
        agreed on deleting line 2, and this could be a synchronization point.
1214
        Line 1 was then considered in isolation, and thought to be deleted on 
1215
        both sides.
1216
1217
        It's better to consider the whole thing as a disagreement region.
1218
        """
1219
        base = """\
1220
            start context
1221
            base line 1
1222
            base line 2
1223
            end context
1224
            """
1225
        a = """\
1226
            start context
1227
            base line 1
1228
            a's replacement line 2
1229
            end context
1230
            """
1231
        b = """\
1232
            start context
1233
            b replaces
1234
            both lines
1235
            end context
1236
            """
1237
        result = """\
1238
            start context
1239
<<<<<<< 
1240
            base line 1
1241
            a's replacement line 2
1242
=======
1243
            b replaces
1244
            both lines
1245
>>>>>>> 
1246
            end context
1247
            """
1248
        self._test_merge_from_strings(base, a, b, result)
1249
1250
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
1251
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1664.2.9 by Aaron Bentley
Ported weave merge test to versionedfile
1252
1253
    def get_file(self, name='foo'):
1254
        return WeaveFile(name, get_transport(self.get_url('.')), create=True)
1255
1256
    def log_contents(self, w):
1257
        self.log('weave is:')
1258
        tmpf = StringIO()
1259
        write_weave(w, tmpf)
1260
        self.log(tmpf.getvalue())
1261
1262
    overlappedInsertExpected = ['aaa', '<<<<<<< ', 'xxx', 'yyy', '=======', 
1263
                                'xxx', '>>>>>>> ', 'bbb']
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1264
1265
1266
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1267
1268
    def test_select_adaptor(self):
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1269
        """Test expected adapters exist."""
1270
        # One scenario for each lookup combination we expect to use.
1271
        # Each is source_kind, requested_kind, adapter class
1272
        scenarios = [
1273
            ('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1274
            ('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1275
            ('knit-annotated-delta-gz', 'knit-delta-gz',
1276
                _mod_knit.DeltaAnnotatedToUnannotated),
1277
            ('knit-annotated-delta-gz', 'fulltext',
1278
                _mod_knit.DeltaAnnotatedToFullText),
1279
            ('knit-annotated-ft-gz', 'knit-ft-gz',
1280
                _mod_knit.FTAnnotatedToUnannotated),
1281
            ('knit-annotated-ft-gz', 'fulltext',
1282
                _mod_knit.FTAnnotatedToFullText),
1283
            ]
1284
        for source, requested, klass in scenarios:
1285
            adapter_factory = versionedfile.adapter_registry.get(
1286
                (source, requested))
1287
            adapter = adapter_factory(None)
1288
            self.assertIsInstance(adapter, klass)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1289
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1290
    def get_knit(self, annotated=True):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1291
        mapper = ConstantMapper('knit')
1292
        transport = self.get_transport()
1293
        return make_file_factory(annotated, mapper)(transport)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1294
1295
    def helpGetBytes(self, f, ft_adapter, delta_adapter):
3350.3.22 by Robert Collins
Review feedback.
1296
        """Grab the interested adapted texts for tests."""
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1297
        # origin is a fulltext
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1298
        entries = f.get_record_stream([('origin',)], 'unordered', False)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1299
        base = entries.next()
4005.3.1 by Robert Collins
Change the signature on VersionedFiles adapters to allow less typing and more flexability inside adapters.
1300
        ft_data = ft_adapter.get_bytes(base)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1301
        # merged is both a delta and multiple parents.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1302
        entries = f.get_record_stream([('merged',)], 'unordered', False)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1303
        merged = entries.next()
4005.3.1 by Robert Collins
Change the signature on VersionedFiles adapters to allow less typing and more flexability inside adapters.
1304
        delta_data = delta_adapter.get_bytes(merged)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1305
        return ft_data, delta_data
1306
1307
    def test_deannotation_noeol(self):
1308
        """Test converting annotated knits to unannotated knits."""
1309
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1310
        f = self.get_knit()
1311
        get_diamond_files(f, 1, trailing_eol=False)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1312
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1313
            _mod_knit.FTAnnotatedToUnannotated(None),
1314
            _mod_knit.DeltaAnnotatedToUnannotated(None))
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1315
        self.assertEqual(
1316
            'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1317
            'origin\n'
1318
            'end origin\n',
1319
            GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1320
        self.assertEqual(
1321
            'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1322
            '1,2,3\nleft\nright\nmerged\nend merged\n',
1323
            GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1324
1325
    def test_deannotation(self):
1326
        """Test converting annotated knits to unannotated knits."""
1327
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1328
        f = self.get_knit()
1329
        get_diamond_files(f, 1)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1330
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1331
            _mod_knit.FTAnnotatedToUnannotated(None),
1332
            _mod_knit.DeltaAnnotatedToUnannotated(None))
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1333
        self.assertEqual(
1334
            'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1335
            'origin\n'
1336
            'end origin\n',
1337
            GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1338
        self.assertEqual(
1339
            'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1340
            '2,2,2\nright\nmerged\nend merged\n',
1341
            GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1342
1343
    def test_annotated_to_fulltext_no_eol(self):
1344
        """Test adapting annotated knits to full texts (for -> weaves)."""
1345
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1346
        f = self.get_knit()
1347
        get_diamond_files(f, 1, trailing_eol=False)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1348
        # Reconstructing a full text requires a backing versioned file, and it
1349
        # must have the base lines requested from it.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1350
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1351
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1352
            _mod_knit.FTAnnotatedToFullText(None),
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1353
            _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1354
        self.assertEqual('origin', ft_data)
1355
        self.assertEqual('base\nleft\nright\nmerged', delta_data)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1356
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1357
            True)], logged_vf.calls)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1358
1359
    def test_annotated_to_fulltext(self):
1360
        """Test adapting annotated knits to full texts (for -> weaves)."""
1361
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1362
        f = self.get_knit()
1363
        get_diamond_files(f, 1)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1364
        # Reconstructing a full text requires a backing versioned file, and it
1365
        # must have the base lines requested from it.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1366
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1367
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1368
            _mod_knit.FTAnnotatedToFullText(None),
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
1369
            _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1370
        self.assertEqual('origin\n', ft_data)
1371
        self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1372
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1373
            True)], logged_vf.calls)
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1374
1375
    def test_unannotated_to_fulltext(self):
1376
        """Test adapting unannotated knits to full texts.
1377
        
1378
        This is used for -> weaves, and for -> annotated knits.
1379
        """
1380
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1381
        f = self.get_knit(annotated=False)
1382
        get_diamond_files(f, 1)
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1383
        # Reconstructing a full text requires a backing versioned file, and it
1384
        # must have the base lines requested from it.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1385
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1386
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1387
            _mod_knit.FTPlainToFullText(None),
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1388
            _mod_knit.DeltaPlainToFullText(logged_vf))
1389
        self.assertEqual('origin\n', ft_data)
1390
        self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1391
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1392
            True)], logged_vf.calls)
3350.3.5 by Robert Collins
Create adapters from plain compressed knit content.
1393
3350.3.6 by Robert Collins
Test EOL behaviour of plain knit record adapters.
1394
    def test_unannotated_to_fulltext_no_eol(self):
1395
        """Test adapting unannotated knits to full texts.
1396
        
1397
        This is used for -> weaves, and for -> annotated knits.
1398
        """
1399
        # we need a full text, and a delta
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1400
        f = self.get_knit(annotated=False)
1401
        get_diamond_files(f, 1, trailing_eol=False)
3350.3.6 by Robert Collins
Test EOL behaviour of plain knit record adapters.
1402
        # Reconstructing a full text requires a backing versioned file, and it
1403
        # must have the base lines requested from it.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1404
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
3350.3.6 by Robert Collins
Test EOL behaviour of plain knit record adapters.
1405
        ft_data, delta_data = self.helpGetBytes(f,
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
1406
            _mod_knit.FTPlainToFullText(None),
3350.3.6 by Robert Collins
Test EOL behaviour of plain knit record adapters.
1407
            _mod_knit.DeltaPlainToFullText(logged_vf))
1408
        self.assertEqual('origin', ft_data)
1409
        self.assertEqual('base\nleft\nright\nmerged', delta_data)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1410
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1411
            True)], logged_vf.calls)
3350.3.6 by Robert Collins
Test EOL behaviour of plain knit record adapters.
1412
3350.6.1 by Robert Collins
* New ``versionedfile.KeyMapper`` interface to abstract out the access to
1413
1414
class TestKeyMapper(TestCaseWithMemoryTransport):
1415
    """Tests for various key mapping logic."""
1416
1417
    def test_identity_mapper(self):
1418
        mapper = versionedfile.ConstantMapper("inventory")
1419
        self.assertEqual("inventory", mapper.map(('foo@ar',)))
1420
        self.assertEqual("inventory", mapper.map(('quux',)))
1421
1422
    def test_prefix_mapper(self):
1423
        #format5: plain
1424
        mapper = versionedfile.PrefixMapper()
1425
        self.assertEqual("file-id", mapper.map(("file-id", "revision-id")))
1426
        self.assertEqual("new-id", mapper.map(("new-id", "revision-id")))
1427
        self.assertEqual(('file-id',), mapper.unmap("file-id"))
1428
        self.assertEqual(('new-id',), mapper.unmap("new-id"))
1429
1430
    def test_hash_prefix_mapper(self):
1431
        #format6: hash + plain
1432
        mapper = versionedfile.HashPrefixMapper()
1433
        self.assertEqual("9b/file-id", mapper.map(("file-id", "revision-id")))
1434
        self.assertEqual("45/new-id", mapper.map(("new-id", "revision-id")))
1435
        self.assertEqual(('file-id',), mapper.unmap("9b/file-id"))
1436
        self.assertEqual(('new-id',), mapper.unmap("45/new-id"))
1437
1438
    def test_hash_escaped_mapper(self):
1439
        #knit1: hash + escaped
1440
        mapper = versionedfile.HashEscapedPrefixMapper()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1441
        self.assertEqual("88/%2520", mapper.map((" ", "revision-id")))
1442
        self.assertEqual("ed/fil%2545-%2549d", mapper.map(("filE-Id",
1443
            "revision-id")))
1444
        self.assertEqual("88/ne%2557-%2549d", mapper.map(("neW-Id",
1445
            "revision-id")))
1446
        self.assertEqual(('filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1447
        self.assertEqual(('neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
3350.6.2 by Robert Collins
Prepare parameterised test environment.
1448
1449
1450
class TestVersionedFiles(TestCaseWithMemoryTransport):
1451
    """Tests for the multiple-file variant of VersionedFile."""
1452
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1453
    def get_versionedfiles(self, relpath='files'):
1454
        transport = self.get_transport(relpath)
1455
        if relpath != '.':
1456
            transport.mkdir('.')
1457
        files = self.factory(transport)
1458
        if self.cleanup is not None:
1459
            self.addCleanup(lambda:self.cleanup(files))
1460
        return files
1461
1462
    def test_annotate(self):
1463
        files = self.get_versionedfiles()
1464
        self.get_diamond_files(files)
1465
        if self.key_length == 1:
1466
            prefix = ()
1467
        else:
1468
            prefix = ('FileA',)
1469
        # introduced full text
1470
        origins = files.annotate(prefix + ('origin',))
1471
        self.assertEqual([
1472
            (prefix + ('origin',), 'origin\n')],
1473
            origins)
1474
        # a delta
1475
        origins = files.annotate(prefix + ('base',))
1476
        self.assertEqual([
1477
            (prefix + ('base',), 'base\n')],
1478
            origins)
1479
        # a merge
1480
        origins = files.annotate(prefix + ('merged',))
1481
        if self.graph:
1482
            self.assertEqual([
1483
                (prefix + ('base',), 'base\n'),
1484
                (prefix + ('left',), 'left\n'),
1485
                (prefix + ('right',), 'right\n'),
1486
                (prefix + ('merged',), 'merged\n')
1487
                ],
1488
                origins)
1489
        else:
1490
            # Without a graph everything is new.
1491
            self.assertEqual([
1492
                (prefix + ('merged',), 'base\n'),
1493
                (prefix + ('merged',), 'left\n'),
1494
                (prefix + ('merged',), 'right\n'),
1495
                (prefix + ('merged',), 'merged\n')
1496
                ],
1497
                origins)
1498
        self.assertRaises(RevisionNotPresent,
1499
            files.annotate, prefix + ('missing-key',))
1500
3350.6.2 by Robert Collins
Prepare parameterised test environment.
1501
    def test_construct(self):
1502
        """Each parameterised test can be constructed on a transport."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1503
        files = self.get_versionedfiles()
1504
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
1505
    def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1506
        nokeys=False):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1507
        return get_diamond_files(files, self.key_length,
1508
            trailing_eol=trailing_eol, nograph=not self.graph,
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
1509
            left_only=left_only, nokeys=nokeys)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1510
1511
    def test_add_lines_return(self):
1512
        files = self.get_versionedfiles()
1513
        # save code by using the stock data insertion helper.
1514
        adds = self.get_diamond_files(files)
1515
        results = []
1516
        # We can only validate the first 2 elements returned from add_lines.
1517
        for add in adds:
1518
            self.assertEqual(3, len(add))
1519
            results.append(add[:2])
1520
        if self.key_length == 1:
1521
            self.assertEqual([
1522
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1523
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1524
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1525
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1526
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1527
                results)
1528
        elif self.key_length == 2:
1529
            self.assertEqual([
1530
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1531
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1532
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1533
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1534
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1535
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1536
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1537
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1538
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1539
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1540
                results)
1541
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
1542
    def test_add_lines_no_key_generates_chk_key(self):
1543
        files = self.get_versionedfiles()
1544
        # save code by using the stock data insertion helper.
1545
        adds = self.get_diamond_files(files, nokeys=True)
1546
        results = []
1547
        # We can only validate the first 2 elements returned from add_lines.
1548
        for add in adds:
1549
            self.assertEqual(3, len(add))
1550
            results.append(add[:2])
1551
        if self.key_length == 1:
1552
            self.assertEqual([
1553
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1554
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1555
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1556
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1557
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1558
                results)
1559
            # Check the added items got CHK keys.
1560
            self.assertEqual(set([
1561
                ('sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1562
                ('sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1563
                ('sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1564
                ('sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1565
                ('sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1566
                ]),
1567
                files.keys())
1568
        elif self.key_length == 2:
1569
            self.assertEqual([
1570
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1571
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1572
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1573
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1574
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1575
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1576
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1577
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1578
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1579
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1580
                results)
1581
            # Check the added items got CHK keys.
1582
            self.assertEqual(set([
1583
                ('FileA', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1584
                ('FileA', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1585
                ('FileA', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1586
                ('FileA', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1587
                ('FileA', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1588
                ('FileB', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1589
                ('FileB', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1590
                ('FileB', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1591
                ('FileB', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1592
                ('FileB', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1593
                ]),
1594
                files.keys())
1595
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1596
    def test_empty_lines(self):
1597
        """Empty files can be stored."""
1598
        f = self.get_versionedfiles()
1599
        key_a = self.get_simple_key('a')
1600
        f.add_lines(key_a, [], [])
1601
        self.assertEqual('',
1602
            f.get_record_stream([key_a], 'unordered', True
1603
                ).next().get_bytes_as('fulltext'))
1604
        key_b = self.get_simple_key('b')
1605
        f.add_lines(key_b, self.get_parents([key_a]), [])
1606
        self.assertEqual('',
1607
            f.get_record_stream([key_b], 'unordered', True
1608
                ).next().get_bytes_as('fulltext'))
1609
1610
    def test_newline_only(self):
1611
        f = self.get_versionedfiles()
1612
        key_a = self.get_simple_key('a')
1613
        f.add_lines(key_a, [], ['\n'])
1614
        self.assertEqual('\n',
1615
            f.get_record_stream([key_a], 'unordered', True
1616
                ).next().get_bytes_as('fulltext'))
1617
        key_b = self.get_simple_key('b')
1618
        f.add_lines(key_b, self.get_parents([key_a]), ['\n'])
1619
        self.assertEqual('\n',
1620
            f.get_record_stream([key_b], 'unordered', True
1621
                ).next().get_bytes_as('fulltext'))
1622
1623
    def test_get_record_stream_empty(self):
1624
        """An empty stream can be requested without error."""
1625
        f = self.get_versionedfiles()
1626
        entries = f.get_record_stream([], 'unordered', False)
1627
        self.assertEqual([], list(entries))
1628
1629
    def assertValidStorageKind(self, storage_kind):
1630
        """Assert that storage_kind is a valid storage_kind."""
1631
        self.assertSubset([storage_kind],
1632
            ['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
1633
             'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1634
             'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
4005.3.6 by Robert Collins
Support delta_closure=True with NetworkRecordStream to transmit deltas over the wire when full text extraction is required on the far end.
1635
             'knit-delta-gz',
1636
             'knit-delta-closure', 'knit-delta-closure-ref'])
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1637
1638
    def capture_stream(self, f, entries, on_seen, parents):
1639
        """Capture a stream for testing."""
1640
        for factory in entries:
1641
            on_seen(factory.key)
1642
            self.assertValidStorageKind(factory.storage_kind)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1643
            self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1644
                factory.sha1)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1645
            self.assertEqual(parents[factory.key], factory.parents)
1646
            self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1647
                str)
1648
1649
    def test_get_record_stream_interface(self):
1650
        """each item in a stream has to provide a regular interface."""
1651
        files = self.get_versionedfiles()
1652
        self.get_diamond_files(files)
1653
        keys, _ = self.get_keys_and_sort_order()
1654
        parent_map = files.get_parent_map(keys)
1655
        entries = files.get_record_stream(keys, 'unordered', False)
1656
        seen = set()
1657
        self.capture_stream(files, entries, seen.add, parent_map)
1658
        self.assertEqual(set(keys), seen)
1659
1660
    def get_simple_key(self, suffix):
1661
        """Return a key for the object under test."""
1662
        if self.key_length == 1:
1663
            return (suffix,)
1664
        else:
1665
            return ('FileA',) + (suffix,)
1666
1667
    def get_keys_and_sort_order(self):
1668
        """Get diamond test keys list, and their sort ordering."""
1669
        if self.key_length == 1:
1670
            keys = [('merged',), ('left',), ('right',), ('base',)]
1671
            sort_order = {('merged',):2, ('left',):1, ('right',):1, ('base',):0}
1672
        else:
1673
            keys = [
1674
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1675
                ('FileA', 'base'),
1676
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1677
                ('FileB', 'base'),
1678
                ]
1679
            sort_order = {
1680
                ('FileA', 'merged'):2, ('FileA', 'left'):1, ('FileA', 'right'):1,
1681
                ('FileA', 'base'):0,
1682
                ('FileB', 'merged'):2, ('FileB', 'left'):1, ('FileB', 'right'):1,
1683
                ('FileB', 'base'):0,
1684
                }
1685
        return keys, sort_order
1686
1687
    def test_get_record_stream_interface_ordered(self):
1688
        """each item in a stream has to provide a regular interface."""
1689
        files = self.get_versionedfiles()
1690
        self.get_diamond_files(files)
1691
        keys, sort_order = self.get_keys_and_sort_order()
1692
        parent_map = files.get_parent_map(keys)
1693
        entries = files.get_record_stream(keys, 'topological', False)
1694
        seen = []
1695
        self.capture_stream(files, entries, seen.append, parent_map)
1696
        self.assertStreamOrder(sort_order, seen, keys)
1697
1698
    def test_get_record_stream_interface_ordered_with_delta_closure(self):
1699
        """each item must be accessible as a fulltext."""
1700
        files = self.get_versionedfiles()
1701
        self.get_diamond_files(files)
1702
        keys, sort_order = self.get_keys_and_sort_order()
1703
        parent_map = files.get_parent_map(keys)
1704
        entries = files.get_record_stream(keys, 'topological', True)
1705
        seen = []
1706
        for factory in entries:
1707
            seen.append(factory.key)
1708
            self.assertValidStorageKind(factory.storage_kind)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1709
            self.assertSubset([factory.sha1],
1710
                [None, files.get_sha1s([factory.key])[factory.key]])
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1711
            self.assertEqual(parent_map[factory.key], factory.parents)
1712
            # self.assertEqual(files.get_text(factory.key),
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
1713
            ft_bytes = factory.get_bytes_as('fulltext')
1714
            self.assertIsInstance(ft_bytes, str)
1715
            chunked_bytes = factory.get_bytes_as('chunked')
1716
            self.assertEqualDiff(ft_bytes, ''.join(chunked_bytes))
1717
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1718
        self.assertStreamOrder(sort_order, seen, keys)
1719
1720
    def assertStreamOrder(self, sort_order, seen, keys):
1721
        self.assertEqual(len(set(seen)), len(keys))
1722
        if self.key_length == 1:
1723
            lows = {():0}
1724
        else:
1725
            lows = {('FileA',):0, ('FileB',):0}
1726
        if not self.graph:
1727
            self.assertEqual(set(keys), set(seen))
1728
        else:
1729
            for key in seen:
1730
                sort_pos = sort_order[key]
1731
                self.assertTrue(sort_pos >= lows[key[:-1]],
1732
                    "Out of order in sorted stream: %r, %r" % (key, seen))
1733
                lows[key[:-1]] = sort_pos
1734
1735
    def test_get_record_stream_unknown_storage_kind_raises(self):
1736
        """Asking for a storage kind that the stream cannot supply raises."""
1737
        files = self.get_versionedfiles()
1738
        self.get_diamond_files(files)
1739
        if self.key_length == 1:
1740
            keys = [('merged',), ('left',), ('right',), ('base',)]
1741
        else:
1742
            keys = [
1743
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1744
                ('FileA', 'base'),
1745
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1746
                ('FileB', 'base'),
1747
                ]
1748
        parent_map = files.get_parent_map(keys)
1749
        entries = files.get_record_stream(keys, 'unordered', False)
1750
        # We track the contents because we should be able to try, fail a
1751
        # particular kind and then ask for one that works and continue.
1752
        seen = set()
1753
        for factory in entries:
1754
            seen.add(factory.key)
1755
            self.assertValidStorageKind(factory.storage_kind)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1756
            self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1757
                factory.sha1)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1758
            self.assertEqual(parent_map[factory.key], factory.parents)
1759
            # currently no stream emits mpdiff
1760
            self.assertRaises(errors.UnavailableRepresentation,
1761
                factory.get_bytes_as, 'mpdiff')
1762
            self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1763
                str)
1764
        self.assertEqual(set(keys), seen)
1765
1766
    def test_get_record_stream_missing_records_are_absent(self):
1767
        files = self.get_versionedfiles()
1768
        self.get_diamond_files(files)
1769
        if self.key_length == 1:
1770
            keys = [('merged',), ('left',), ('right',), ('absent',), ('base',)]
1771
        else:
1772
            keys = [
1773
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1774
                ('FileA', 'absent'), ('FileA', 'base'),
1775
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1776
                ('FileB', 'absent'), ('FileB', 'base'),
1777
                ('absent', 'absent'),
1778
                ]
1779
        parent_map = files.get_parent_map(keys)
1780
        entries = files.get_record_stream(keys, 'unordered', False)
1781
        self.assertAbsentRecord(files, keys, parent_map, entries)
1782
        entries = files.get_record_stream(keys, 'topological', False)
1783
        self.assertAbsentRecord(files, keys, parent_map, entries)
1784
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1785
    def assertRecordHasContent(self, record, bytes):
1786
        """Assert that record has the bytes bytes."""
1787
        self.assertEqual(bytes, record.get_bytes_as('fulltext'))
4005.3.7 by Robert Collins
Review feedback.
1788
        self.assertEqual(bytes, ''.join(record.get_bytes_as('chunked')))
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1789
4005.3.2 by Robert Collins
First passing NetworkRecordStream test - a fulltext from any record type which isn't a chunked or fulltext can be serialised and deserialised successfully.
1790
    def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
1791
        files = self.get_versionedfiles()
1792
        key = self.get_simple_key('foo')
1793
        files.add_lines(key, (), ['my text\n', 'content'])
1794
        stream = files.get_record_stream([key], 'unordered', False)
1795
        record = stream.next()
1796
        if record.storage_kind in ('chunked', 'fulltext'):
1797
            # chunked and fulltext representations are for direct use not wire
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1798
            # serialisation: check they are able to be used directly. To send
1799
            # such records over the wire translation will be needed.
1800
            self.assertRecordHasContent(record, "my text\ncontent")
4005.3.2 by Robert Collins
First passing NetworkRecordStream test - a fulltext from any record type which isn't a chunked or fulltext can be serialised and deserialised successfully.
1801
        else:
1802
            bytes = [record.get_bytes_as(record.storage_kind)]
1803
            network_stream = versionedfile.NetworkRecordStream(bytes).read()
1804
            source_record = record
1805
            records = []
1806
            for record in network_stream:
1807
                records.append(record)
1808
                self.assertEqual(source_record.storage_kind,
1809
                    record.storage_kind)
1810
                self.assertEqual(source_record.parents, record.parents)
1811
                self.assertEqual(
1812
                    source_record.get_bytes_as(source_record.storage_kind),
1813
                    record.get_bytes_as(record.storage_kind))
1814
            self.assertEqual(1, len(records))
1815
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1816
    def assertStreamMetaEqual(self, records, expected, stream):
1817
        """Assert that streams expected and stream have the same records.
1818
        
1819
        :param records: A list to collect the seen records.
1820
        :return: A generator of the records in stream.
1821
        """
1822
        # We make assertions during copying to catch things early for
1823
        # easier debugging.
1824
        for record, ref_record in izip(stream, expected):
1825
            records.append(record)
1826
            self.assertEqual(ref_record.key, record.key)
1827
            self.assertEqual(ref_record.storage_kind, record.storage_kind)
1828
            self.assertEqual(ref_record.parents, record.parents)
1829
            yield record
1830
1831
    def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
1832
        stream):
1833
        """Convert a stream to a bytes iterator.
1834
1835
        :param skipped_records: A list with one element to increment when a
1836
            record is skipped.
1837
        :param full_texts: A dict from key->fulltext representation, for 
1838
            checking chunked or fulltext stored records.
1839
        :param stream: A record_stream.
1840
        :return: An iterator over the bytes of each record.
1841
        """
1842
        for record in stream:
1843
            if record.storage_kind in ('chunked', 'fulltext'):
1844
                skipped_records[0] += 1
1845
                # check the content is correct for direct use.
1846
                self.assertRecordHasContent(record, full_texts[record.key])
1847
            else:
1848
                yield record.get_bytes_as(record.storage_kind)
1849
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1850
    def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
1851
        files = self.get_versionedfiles()
1852
        target_files = self.get_versionedfiles('target')
1853
        key = self.get_simple_key('ft')
1854
        key_delta = self.get_simple_key('delta')
1855
        files.add_lines(key, (), ['my text\n', 'content'])
1856
        if self.graph:
1857
            delta_parents = (key,)
1858
        else:
1859
            delta_parents = ()
1860
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1861
        local = files.get_record_stream([key, key_delta], 'unordered', False)
1862
        ref = files.get_record_stream([key, key_delta], 'unordered', False)
1863
        skipped_records = [0]
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1864
        full_texts = {
1865
            key: "my text\ncontent",
1866
            key_delta: "different\ncontent\n",
1867
            }
1868
        byte_stream = self.stream_to_bytes_or_skip_counter(
1869
            skipped_records, full_texts, local)
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1870
        network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1871
        records = []
1872
        # insert the stream from the network into a versioned files object so we can
1873
        # check the content was carried across correctly without doing delta
1874
        # inspection.
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1875
        target_files.insert_record_stream(
1876
            self.assertStreamMetaEqual(records, ref, network_stream))
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1877
        # No duplicates on the wire thank you!
1878
        self.assertEqual(2, len(records) + skipped_records[0])
1879
        if len(records):
4005.3.4 by Robert Collins
Test copying just a delta over the wire.
1880
            # if any content was copied it all must have all been.
1881
            self.assertIdenticalVersionedFile(files, target_files)
1882
1883
    def test_get_record_stream_native_formats_are_wire_ready_delta(self):
1884
        # copy a delta over the wire
1885
        files = self.get_versionedfiles()
1886
        target_files = self.get_versionedfiles('target')
1887
        key = self.get_simple_key('ft')
1888
        key_delta = self.get_simple_key('delta')
1889
        files.add_lines(key, (), ['my text\n', 'content'])
1890
        if self.graph:
1891
            delta_parents = (key,)
1892
        else:
1893
            delta_parents = ()
1894
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1895
        # Copy the basis text across so we can reconstruct the delta during
1896
        # insertion into target.
4005.3.4 by Robert Collins
Test copying just a delta over the wire.
1897
        target_files.insert_record_stream(files.get_record_stream([key],
1898
            'unordered', False))
1899
        local = files.get_record_stream([key_delta], 'unordered', False)
1900
        ref = files.get_record_stream([key_delta], 'unordered', False)
1901
        skipped_records = [0]
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1902
        full_texts = {
1903
            key_delta: "different\ncontent\n",
1904
            }
1905
        byte_stream = self.stream_to_bytes_or_skip_counter(
1906
            skipped_records, full_texts, local)
4005.3.4 by Robert Collins
Test copying just a delta over the wire.
1907
        network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1908
        records = []
1909
        # insert the stream from the network into a versioned files object so we can
1910
        # check the content was carried across correctly without doing delta
1911
        # inspection during check_stream.
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1912
        target_files.insert_record_stream(
1913
            self.assertStreamMetaEqual(records, ref, network_stream))
4005.3.4 by Robert Collins
Test copying just a delta over the wire.
1914
        # No duplicates on the wire thank you!
1915
        self.assertEqual(1, len(records) + skipped_records[0])
1916
        if len(records):
1917
            # if any content was copied it all must have all been
4005.3.3 by Robert Collins
Test NetworkRecordStream with delta'd texts.
1918
            self.assertIdenticalVersionedFile(files, target_files)
1919
4005.3.5 by Robert Collins
Interface level test for using delta_closure=True over the network.
1920
    def test_get_record_stream_wire_ready_delta_closure_included(self):
1921
        # copy a delta over the wire with the ability to get its full text.
1922
        files = self.get_versionedfiles()
1923
        key = self.get_simple_key('ft')
1924
        key_delta = self.get_simple_key('delta')
1925
        files.add_lines(key, (), ['my text\n', 'content'])
1926
        if self.graph:
1927
            delta_parents = (key,)
1928
        else:
1929
            delta_parents = ()
1930
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1931
        local = files.get_record_stream([key_delta], 'unordered', True)
1932
        ref = files.get_record_stream([key_delta], 'unordered', True)
1933
        skipped_records = [0]
1934
        full_texts = {
1935
            key_delta: "different\ncontent\n",
1936
            }
1937
        byte_stream = self.stream_to_bytes_or_skip_counter(
1938
            skipped_records, full_texts, local)
1939
        network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1940
        records = []
1941
        # insert the stream from the network into a versioned files object so we can
1942
        # check the content was carried across correctly without doing delta
1943
        # inspection during check_stream.
1944
        for record in self.assertStreamMetaEqual(records, ref, network_stream):
1945
            # we have to be able to get the full text out:
1946
            self.assertRecordHasContent(record, full_texts[record.key])
1947
        # No duplicates on the wire thank you!
1948
        self.assertEqual(1, len(records) + skipped_records[0])
1949
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1950
    def assertAbsentRecord(self, files, keys, parents, entries):
1951
        """Helper for test_get_record_stream_missing_records_are_absent."""
1952
        seen = set()
1953
        for factory in entries:
1954
            seen.add(factory.key)
1955
            if factory.key[-1] == 'absent':
1956
                self.assertEqual('absent', factory.storage_kind)
1957
                self.assertEqual(None, factory.sha1)
1958
                self.assertEqual(None, factory.parents)
1959
            else:
1960
                self.assertValidStorageKind(factory.storage_kind)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1961
                self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1962
                    factory.sha1)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1963
                self.assertEqual(parents[factory.key], factory.parents)
1964
                self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1965
                    str)
1966
        self.assertEqual(set(keys), seen)
1967
1968
    def test_filter_absent_records(self):
1969
        """Requested missing records can be filter trivially."""
1970
        files = self.get_versionedfiles()
1971
        self.get_diamond_files(files)
1972
        keys, _ = self.get_keys_and_sort_order()
1973
        parent_map = files.get_parent_map(keys)
1974
        # Add an absent record in the middle of the present keys. (We don't ask
1975
        # for just absent keys to ensure that content before and after the
1976
        # absent keys is still delivered).
1977
        present_keys = list(keys)
1978
        if self.key_length == 1:
1979
            keys.insert(2, ('extra',))
1980
        else:
1981
            keys.insert(2, ('extra', 'extra'))
1982
        entries = files.get_record_stream(keys, 'unordered', False)
1983
        seen = set()
1984
        self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
1985
            parent_map)
1986
        self.assertEqual(set(present_keys), seen)
1987
1988
    def get_mapper(self):
1989
        """Get a mapper suitable for the key length of the test interface."""
1990
        if self.key_length == 1:
1991
            return ConstantMapper('source')
1992
        else:
1993
            return HashEscapedPrefixMapper()
1994
1995
    def get_parents(self, parents):
1996
        """Get parents, taking self.graph into consideration."""
1997
        if self.graph:
1998
            return parents
1999
        else:
2000
            return None
2001
2002
    def test_get_parent_map(self):
2003
        files = self.get_versionedfiles()
2004
        if self.key_length == 1:
2005
            parent_details = [
2006
                (('r0',), self.get_parents(())),
2007
                (('r1',), self.get_parents((('r0',),))),
2008
                (('r2',), self.get_parents(())),
2009
                (('r3',), self.get_parents(())),
2010
                (('m',), self.get_parents((('r0',),('r1',),('r2',),('r3',)))),
2011
                ]
2012
        else:
2013
            parent_details = [
2014
                (('FileA', 'r0'), self.get_parents(())),
2015
                (('FileA', 'r1'), self.get_parents((('FileA', 'r0'),))),
2016
                (('FileA', 'r2'), self.get_parents(())),
2017
                (('FileA', 'r3'), self.get_parents(())),
2018
                (('FileA', 'm'), self.get_parents((('FileA', 'r0'),
2019
                    ('FileA', 'r1'), ('FileA', 'r2'), ('FileA', 'r3')))),
2020
                ]
2021
        for key, parents in parent_details:
2022
            files.add_lines(key, parents, [])
2023
            # immediately after adding it should be queryable.
2024
            self.assertEqual({key:parents}, files.get_parent_map([key]))
2025
        # We can ask for an empty set
2026
        self.assertEqual({}, files.get_parent_map([]))
2027
        # We can ask for many keys
2028
        all_parents = dict(parent_details)
2029
        self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2030
        # Absent keys are just not included in the result.
2031
        keys = all_parents.keys()
2032
        if self.key_length == 1:
2033
            keys.insert(1, ('missing',))
2034
        else:
2035
            keys.insert(1, ('missing', 'missing'))
2036
        # Absent keys are just ignored
2037
        self.assertEqual(all_parents, files.get_parent_map(keys))
2038
2039
    def test_get_sha1s(self):
2040
        files = self.get_versionedfiles()
2041
        self.get_diamond_files(files)
2042
        if self.key_length == 1:
2043
            keys = [('base',), ('origin',), ('left',), ('merged',), ('right',)]
2044
        else:
2045
            # ask for shas from different prefixes.
2046
            keys = [
2047
                ('FileA', 'base'), ('FileB', 'origin'), ('FileA', 'left'),
2048
                ('FileA', 'merged'), ('FileB', 'right'),
2049
                ]
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
2050
        self.assertEqual({
2051
            keys[0]: '51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2052
            keys[1]: '00e364d235126be43292ab09cb4686cf703ddc17',
2053
            keys[2]: 'a8478686da38e370e32e42e8a0c220e33ee9132f',
2054
            keys[3]: 'ed8bce375198ea62444dc71952b22cfc2b09226d',
2055
            keys[4]: '9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2056
            },
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2057
            files.get_sha1s(keys))
2058
        
2059
    def test_insert_record_stream_empty(self):
2060
        """Inserting an empty record stream should work."""
2061
        files = self.get_versionedfiles()
2062
        files.insert_record_stream([])
2063
2064
    def assertIdenticalVersionedFile(self, expected, actual):
2065
        """Assert that left and right have the same contents."""
2066
        self.assertEqual(set(actual.keys()), set(expected.keys()))
2067
        actual_parents = actual.get_parent_map(actual.keys())
2068
        if self.graph:
2069
            self.assertEqual(actual_parents, expected.get_parent_map(expected.keys()))
2070
        else:
2071
            for key, parents in actual_parents.items():
2072
                self.assertEqual(None, parents)
2073
        for key in actual.keys():
2074
            actual_text = actual.get_record_stream(
2075
                [key], 'unordered', True).next().get_bytes_as('fulltext')
2076
            expected_text = expected.get_record_stream(
2077
                [key], 'unordered', True).next().get_bytes_as('fulltext')
2078
            self.assertEqual(actual_text, expected_text)
2079
2080
    def test_insert_record_stream_fulltexts(self):
2081
        """Any file should accept a stream of fulltexts."""
2082
        files = self.get_versionedfiles()
2083
        mapper = self.get_mapper()
2084
        source_transport = self.get_transport('source')
2085
        source_transport.mkdir('.')
2086
        # weaves always output fulltexts.
2087
        source = make_versioned_files_factory(WeaveFile, mapper)(
2088
            source_transport)
2089
        self.get_diamond_files(source, trailing_eol=False)
2090
        stream = source.get_record_stream(source.keys(), 'topological',
2091
            False)
2092
        files.insert_record_stream(stream)
2093
        self.assertIdenticalVersionedFile(source, files)
2094
2095
    def test_insert_record_stream_fulltexts_noeol(self):
2096
        """Any file should accept a stream of fulltexts."""
2097
        files = self.get_versionedfiles()
2098
        mapper = self.get_mapper()
2099
        source_transport = self.get_transport('source')
2100
        source_transport.mkdir('.')
2101
        # weaves always output fulltexts.
2102
        source = make_versioned_files_factory(WeaveFile, mapper)(
2103
            source_transport)
2104
        self.get_diamond_files(source, trailing_eol=False)
2105
        stream = source.get_record_stream(source.keys(), 'topological',
2106
            False)
2107
        files.insert_record_stream(stream)
2108
        self.assertIdenticalVersionedFile(source, files)
2109
2110
    def test_insert_record_stream_annotated_knits(self):
2111
        """Any file should accept a stream from plain knits."""
2112
        files = self.get_versionedfiles()
2113
        mapper = self.get_mapper()
2114
        source_transport = self.get_transport('source')
2115
        source_transport.mkdir('.')
2116
        source = make_file_factory(True, mapper)(source_transport)
2117
        self.get_diamond_files(source)
2118
        stream = source.get_record_stream(source.keys(), 'topological',
2119
            False)
2120
        files.insert_record_stream(stream)
2121
        self.assertIdenticalVersionedFile(source, files)
2122
2123
    def test_insert_record_stream_annotated_knits_noeol(self):
2124
        """Any file should accept a stream from plain knits."""
2125
        files = self.get_versionedfiles()
2126
        mapper = self.get_mapper()
2127
        source_transport = self.get_transport('source')
2128
        source_transport.mkdir('.')
2129
        source = make_file_factory(True, mapper)(source_transport)
2130
        self.get_diamond_files(source, trailing_eol=False)
2131
        stream = source.get_record_stream(source.keys(), 'topological',
2132
            False)
2133
        files.insert_record_stream(stream)
2134
        self.assertIdenticalVersionedFile(source, files)
2135
2136
    def test_insert_record_stream_plain_knits(self):
2137
        """Any file should accept a stream from plain knits."""
2138
        files = self.get_versionedfiles()
2139
        mapper = self.get_mapper()
2140
        source_transport = self.get_transport('source')
2141
        source_transport.mkdir('.')
2142
        source = make_file_factory(False, mapper)(source_transport)
2143
        self.get_diamond_files(source)
2144
        stream = source.get_record_stream(source.keys(), 'topological',
2145
            False)
2146
        files.insert_record_stream(stream)
2147
        self.assertIdenticalVersionedFile(source, files)
2148
2149
    def test_insert_record_stream_plain_knits_noeol(self):
2150
        """Any file should accept a stream from plain knits."""
2151
        files = self.get_versionedfiles()
2152
        mapper = self.get_mapper()
2153
        source_transport = self.get_transport('source')
2154
        source_transport.mkdir('.')
2155
        source = make_file_factory(False, mapper)(source_transport)
2156
        self.get_diamond_files(source, trailing_eol=False)
2157
        stream = source.get_record_stream(source.keys(), 'topological',
2158
            False)
2159
        files.insert_record_stream(stream)
2160
        self.assertIdenticalVersionedFile(source, files)
2161
2162
    def test_insert_record_stream_existing_keys(self):
2163
        """Inserting keys already in a file should not error."""
2164
        files = self.get_versionedfiles()
2165
        source = self.get_versionedfiles('source')
2166
        self.get_diamond_files(source)
2167
        # insert some keys into f.
2168
        self.get_diamond_files(files, left_only=True)
2169
        stream = source.get_record_stream(source.keys(), 'topological',
2170
            False)
2171
        files.insert_record_stream(stream)
2172
        self.assertIdenticalVersionedFile(source, files)
2173
2174
    def test_insert_record_stream_missing_keys(self):
2175
        """Inserting a stream with absent keys should raise an error."""
2176
        files = self.get_versionedfiles()
2177
        source = self.get_versionedfiles('source')
2178
        stream = source.get_record_stream([('missing',) * self.key_length],
2179
            'topological', False)
2180
        self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2181
            stream)
2182
2183
    def test_insert_record_stream_out_of_order(self):
2184
        """An out of order stream can either error or work."""
2185
        files = self.get_versionedfiles()
2186
        source = self.get_versionedfiles('source')
2187
        self.get_diamond_files(source)
2188
        if self.key_length == 1:
2189
            origin_keys = [('origin',)]
2190
            end_keys = [('merged',), ('left',)]
2191
            start_keys = [('right',), ('base',)]
2192
        else:
2193
            origin_keys = [('FileA', 'origin'), ('FileB', 'origin')]
2194
            end_keys = [('FileA', 'merged',), ('FileA', 'left',),
2195
                ('FileB', 'merged',), ('FileB', 'left',)]
2196
            start_keys = [('FileA', 'right',), ('FileA', 'base',),
2197
                ('FileB', 'right',), ('FileB', 'base',)]
2198
        origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
2199
        end_entries = source.get_record_stream(end_keys, 'topological', False)
2200
        start_entries = source.get_record_stream(start_keys, 'topological', False)
2201
        entries = chain(origin_entries, end_entries, start_entries)
2202
        try:
2203
            files.insert_record_stream(entries)
2204
        except RevisionNotPresent:
2205
            # Must not have corrupted the file.
2206
            files.check()
2207
        else:
2208
            self.assertIdenticalVersionedFile(source, files)
2209
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2210
    def get_knit_delta_source(self):
2211
        """Get a source that can produce a stream with knit delta records,
2212
        regardless of this test's scenario.
2213
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2214
        mapper = self.get_mapper()
2215
        source_transport = self.get_transport('source')
2216
        source_transport.mkdir('.')
2217
        source = make_file_factory(False, mapper)(source_transport)
4009.3.1 by Andrew Bennetts
Fix test_insert_record_stream_delta_missing_basis_no_corruption to test what it claims to, and fix KnitVersionedFiles.get_record_stream to match the expected exception.
2218
        get_diamond_files(source, self.key_length, trailing_eol=True,
2219
            nograph=False, left_only=False)
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2220
        return source
2221
2222
    def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2223
        """Insertion where a needed basis is not included notifies the caller
2224
        of the missing basis.  In the meantime a record missing its basis is
2225
        not added.
2226
        """
2227
        source = self.get_knit_delta_source()
4009.3.7 by Andrew Bennetts
Most tests passing.
2228
        keys = [self.get_simple_key('origin'), self.get_simple_key('merged')]
2229
        entries = source.get_record_stream(keys, 'unordered', False)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2230
        files = self.get_versionedfiles()
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2231
        if self.support_partial_insertion:
4009.3.12 by Robert Collins
Polish on inserting record streams with missing compression parents.
2232
            self.assertEqual([],
2233
                list(files.get_missing_compression_parent_keys()))
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2234
            files.insert_record_stream(entries)
2235
            missing_bases = files.get_missing_compression_parent_keys()
2236
            self.assertEqual(set([self.get_simple_key('left')]),
2237
                set(missing_bases))
4009.3.7 by Andrew Bennetts
Most tests passing.
2238
            self.assertEqual(set(keys), set(files.get_parent_map(keys)))
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2239
        else:
2240
            self.assertRaises(
2241
                errors.RevisionNotPresent, files.insert_record_stream, entries)
4009.3.7 by Andrew Bennetts
Most tests passing.
2242
            files.check()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2243
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2244
    def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2245
        """Insertion where a needed basis is not included notifies the caller
2246
        of the missing basis.  That basis can be added in a second
2247
        insert_record_stream call that does not need to repeat records present
4009.3.3 by Andrew Bennetts
Add docstrings.
2248
        in the previous stream.  The record(s) that required that basis are
2249
        fully inserted once their basis is no longer missing.
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2250
        """
2251
        if not self.support_partial_insertion:
2252
            raise TestNotApplicable(
2253
                'versioned file scenario does not support partial insertion')
2254
        source = self.get_knit_delta_source()
2255
        entries = source.get_record_stream([self.get_simple_key('origin'),
2256
            self.get_simple_key('merged')], 'unordered', False)
2257
        files = self.get_versionedfiles()
2258
        files.insert_record_stream(entries)
2259
        missing_bases = files.get_missing_compression_parent_keys()
2260
        self.assertEqual(set([self.get_simple_key('left')]),
2261
            set(missing_bases))
4009.3.7 by Andrew Bennetts
Most tests passing.
2262
        # 'merged' is inserted (although a commit of a write group involving
2263
        # this versionedfiles would fail).
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2264
        merged_key = self.get_simple_key('merged')
4009.3.7 by Andrew Bennetts
Most tests passing.
2265
        self.assertEqual(
2266
            [merged_key], files.get_parent_map([merged_key]).keys())
2267
        # Add the full delta closure of the missing records
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2268
        missing_entries = source.get_record_stream(
4009.3.7 by Andrew Bennetts
Most tests passing.
2269
            missing_bases, 'unordered', True)
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2270
        files.insert_record_stream(missing_entries)
4009.3.7 by Andrew Bennetts
Most tests passing.
2271
        # Now 'merged' is fully inserted (and a commit would succeed).
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2272
        self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2273
        self.assertEqual(
2274
            [merged_key], files.get_parent_map([merged_key]).keys())
4009.3.7 by Andrew Bennetts
Most tests passing.
2275
        files.check()
4009.3.2 by Andrew Bennetts
Add test_insert_record_stream_delta_missing_basis_can_be_added_later.
2276
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2277
    def test_iter_lines_added_or_present_in_keys(self):
2278
        # test that we get at least an equalset of the lines added by
2279
        # versions in the store.
2280
        # the ordering here is to make a tree so that dumb searches have
2281
        # more changes to muck up.
2282
2283
        class InstrumentedProgress(progress.DummyProgress):
2284
2285
            def __init__(self):
2286
2287
                progress.DummyProgress.__init__(self)
2288
                self.updates = []
2289
2290
            def update(self, msg=None, current=None, total=None):
2291
                self.updates.append((msg, current, total))
2292
2293
        files = self.get_versionedfiles()
2294
        # add a base to get included
2295
        files.add_lines(self.get_simple_key('base'), (), ['base\n'])
2296
        # add a ancestor to be included on one side
2297
        files.add_lines(self.get_simple_key('lancestor'), (), ['lancestor\n'])
2298
        # add a ancestor to be included on the other side
2299
        files.add_lines(self.get_simple_key('rancestor'),
2300
            self.get_parents([self.get_simple_key('base')]), ['rancestor\n'])
2301
        # add a child of rancestor with no eofile-nl
2302
        files.add_lines(self.get_simple_key('child'),
2303
            self.get_parents([self.get_simple_key('rancestor')]),
2304
            ['base\n', 'child\n'])
2305
        # add a child of lancestor and base to join the two roots
2306
        files.add_lines(self.get_simple_key('otherchild'),
2307
            self.get_parents([self.get_simple_key('lancestor'),
2308
                self.get_simple_key('base')]),
2309
            ['base\n', 'lancestor\n', 'otherchild\n'])
2310
        def iter_with_keys(keys, expected):
2311
            # now we need to see what lines are returned, and how often.
2312
            lines = {}
2313
            progress = InstrumentedProgress()
2314
            # iterate over the lines
2315
            for line in files.iter_lines_added_or_present_in_keys(keys,
2316
                pb=progress):
2317
                lines.setdefault(line, 0)
2318
                lines[line] += 1
2319
            if []!= progress.updates:
2320
                self.assertEqual(expected, progress.updates)
2321
            return lines
2322
        lines = iter_with_keys(
2323
            [self.get_simple_key('child'), self.get_simple_key('otherchild')],
2324
            [('Walking content.', 0, 2),
2325
             ('Walking content.', 1, 2),
2326
             ('Walking content.', 2, 2)])
2327
        # we must see child and otherchild
2328
        self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2329
        self.assertTrue(
2330
            lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2331
        # we dont care if we got more than that.
2332
        
2333
        # test all lines
2334
        lines = iter_with_keys(files.keys(),
2335
            [('Walking content.', 0, 5),
2336
             ('Walking content.', 1, 5),
2337
             ('Walking content.', 2, 5),
2338
             ('Walking content.', 3, 5),
2339
             ('Walking content.', 4, 5),
2340
             ('Walking content.', 5, 5)])
2341
        # all lines must be seen at least once
2342
        self.assertTrue(lines[('base\n', self.get_simple_key('base'))] > 0)
2343
        self.assertTrue(
2344
            lines[('lancestor\n', self.get_simple_key('lancestor'))] > 0)
2345
        self.assertTrue(
2346
            lines[('rancestor\n', self.get_simple_key('rancestor'))] > 0)
2347
        self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2348
        self.assertTrue(
2349
            lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2350
2351
    def test_make_mpdiffs(self):
2352
        from bzrlib import multiparent
2353
        files = self.get_versionedfiles('source')
2354
        # add texts that should trip the knit maximum delta chain threshold
2355
        # as well as doing parallel chains of data in knits.
2356
        # this is done by two chains of 25 insertions
2357
        files.add_lines(self.get_simple_key('base'), [], ['line\n'])
2358
        files.add_lines(self.get_simple_key('noeol'),
2359
            self.get_parents([self.get_simple_key('base')]), ['line'])
2360
        # detailed eol tests:
2361
        # shared last line with parent no-eol
2362
        files.add_lines(self.get_simple_key('noeolsecond'),
2363
            self.get_parents([self.get_simple_key('noeol')]),
2364
                ['line\n', 'line'])
2365
        # differing last line with parent, both no-eol
2366
        files.add_lines(self.get_simple_key('noeolnotshared'),
2367
            self.get_parents([self.get_simple_key('noeolsecond')]),
2368
                ['line\n', 'phone'])
2369
        # add eol following a noneol parent, change content
2370
        files.add_lines(self.get_simple_key('eol'),
2371
            self.get_parents([self.get_simple_key('noeol')]), ['phone\n'])
2372
        # add eol following a noneol parent, no change content
2373
        files.add_lines(self.get_simple_key('eolline'),
2374
            self.get_parents([self.get_simple_key('noeol')]), ['line\n'])
2375
        # noeol with no parents:
2376
        files.add_lines(self.get_simple_key('noeolbase'), [], ['line'])
2377
        # noeol preceeding its leftmost parent in the output:
2378
        # this is done by making it a merge of two parents with no common
2379
        # anestry: noeolbase and noeol with the 
2380
        # later-inserted parent the leftmost.
2381
        files.add_lines(self.get_simple_key('eolbeforefirstparent'),
2382
            self.get_parents([self.get_simple_key('noeolbase'),
2383
                self.get_simple_key('noeol')]),
2384
            ['line'])
2385
        # two identical eol texts
2386
        files.add_lines(self.get_simple_key('noeoldup'),
2387
            self.get_parents([self.get_simple_key('noeol')]), ['line'])
2388
        next_parent = self.get_simple_key('base')
2389
        text_name = 'chain1-'
2390
        text = ['line\n']
2391
        sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2392
                 1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2393
                 2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2394
                 3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2395
                 4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
2396
                 5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
2397
                 6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2398
                 7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2399
                 8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
2400
                 9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2401
                 10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2402
                 11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2403
                 12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
2404
                 13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2405
                 14:'2c4b1736566b8ca6051e668de68650686a3922f2',
2406
                 15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2407
                 16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
2408
                 17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2409
                 18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
2410
                 19:'1ebed371807ba5935958ad0884595126e8c4e823',
2411
                 20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2412
                 21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2413
                 22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2414
                 23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2415
                 24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2416
                 25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2417
                 }
2418
        for depth in range(26):
2419
            new_version = self.get_simple_key(text_name + '%s' % depth)
2420
            text = text + ['line\n']
2421
            files.add_lines(new_version, self.get_parents([next_parent]), text)
2422
            next_parent = new_version
2423
        next_parent = self.get_simple_key('base')
2424
        text_name = 'chain2-'
2425
        text = ['line\n']
2426
        for depth in range(26):
2427
            new_version = self.get_simple_key(text_name + '%s' % depth)
2428
            text = text + ['line\n']
2429
            files.add_lines(new_version, self.get_parents([next_parent]), text)
2430
            next_parent = new_version
2431
        target = self.get_versionedfiles('target')
2432
        for key in multiparent.topo_iter_keys(files, files.keys()):
2433
            mpdiff = files.make_mpdiffs([key])[0]
2434
            parents = files.get_parent_map([key])[key] or []
2435
            target.add_mpdiffs(
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
2436
                [(key, parents, files.get_sha1s([key])[key], mpdiff)])
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2437
            self.assertEqualDiff(
2438
                files.get_record_stream([key], 'unordered',
2439
                    True).next().get_bytes_as('fulltext'),
2440
                target.get_record_stream([key], 'unordered',
2441
                    True).next().get_bytes_as('fulltext')
2442
                )
2443
2444
    def test_keys(self):
2445
        # While use is discouraged, versions() is still needed by aspects of
2446
        # bzr.
2447
        files = self.get_versionedfiles()
2448
        self.assertEqual(set(), set(files.keys()))
2449
        if self.key_length == 1:
2450
            key = ('foo',)
2451
        else:
2452
            key = ('foo', 'bar',)
2453
        files.add_lines(key, (), [])
2454
        self.assertEqual(set([key]), set(files.keys()))
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
2455
2456
2457
class VirtualVersionedFilesTests(TestCase):
2458
    """Basic tests for the VirtualVersionedFiles implementations."""
2459
2460
    def _get_parent_map(self, keys):
2461
        ret = {}
2462
        for k in keys:
2463
            if k in self._parent_map:
2464
                ret[k] = self._parent_map[k]
2465
        return ret
2466
2467
    def setUp(self):
2468
        TestCase.setUp(self)
2469
        self._lines = {}
2470
        self._parent_map = {}
2471
        self.texts = VirtualVersionedFiles(self._get_parent_map, 
2472
                                           self._lines.get)
2473
2474
    def test_add_lines(self):
2475
        self.assertRaises(NotImplementedError, 
2476
                self.texts.add_lines, "foo", [], [])
2477
2478
    def test_add_mpdiffs(self):
2479
        self.assertRaises(NotImplementedError, 
2480
                self.texts.add_mpdiffs, [])
2481
2482
    def test_check(self):
2483
        self.assertTrue(self.texts.check())
2484
2485
    def test_insert_record_stream(self):
2486
        self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2487
                          [])
2488
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
2489
    def test_get_sha1s_nonexistent(self):
2490
        self.assertEquals({}, self.texts.get_sha1s([("NONEXISTENT",)]))
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
2491
2492
    def test_get_sha1s(self):
2493
        self._lines["key"] = ["dataline1", "dataline2"]
2494
        self.assertEquals({("key",): osutils.sha_strings(self._lines["key"])},
2495
                           self.texts.get_sha1s([("key",)]))
2496
2497
    def test_get_parent_map(self):
2498
        self._parent_map = {"G": ("A", "B")}
2499
        self.assertEquals({("G",): (("A",),("B",))}, 
2500
                          self.texts.get_parent_map([("G",), ("L",)]))
2501
2502
    def test_get_record_stream(self):
2503
        self._lines["A"] = ["FOO", "BAR"]
2504
        it = self.texts.get_record_stream([("A",)], "unordered", True)
2505
        record = it.next()
3890.2.2 by John Arbash Meinel
Change the signature to report the storage kind as 'chunked'
2506
        self.assertEquals("chunked", record.storage_kind)
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
2507
        self.assertEquals("FOOBAR", record.get_bytes_as("fulltext"))
3890.2.2 by John Arbash Meinel
Change the signature to report the storage kind as 'chunked'
2508
        self.assertEquals(["FOO", "BAR"], record.get_bytes_as("chunked"))
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
2509
2510
    def test_get_record_stream_absent(self):
2511
        it = self.texts.get_record_stream([("A",)], "unordered", True)
2512
        record = it.next()
2513
        self.assertEquals("absent", record.storage_kind)
2514
3949.4.1 by Jelmer Vernooij
Implement VirtualVersionedFiles.iter_lines_added_or_present_in_keys.
2515
    def test_iter_lines_added_or_present_in_keys(self):
2516
        self._lines["A"] = ["FOO", "BAR"]
2517
        self._lines["B"] = ["HEY"]
2518
        self._lines["C"] = ["Alberta"]
2519
        it = self.texts.iter_lines_added_or_present_in_keys([("A",), ("B",)])
2520
        self.assertEquals(sorted([("FOO", "A"), ("BAR", "A"), ("HEY", "B")]), 
2521
            sorted(list(it)))
2522
3871.4.1 by John Arbash Meinel
Add a VFDecorator that can yield records in a specified order
2523
2524
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2525
2526
    def get_ordering_vf(self, key_priority):
2527
        builder = self.make_branch_builder('test')
2528
        builder.start_series()
2529
        builder.build_snapshot('A', None, [
2530
            ('add', ('', 'TREE_ROOT', 'directory', None))])
2531
        builder.build_snapshot('B', ['A'], [])
2532
        builder.build_snapshot('C', ['B'], [])
2533
        builder.build_snapshot('D', ['C'], [])
2534
        builder.finish_series()
2535
        b = builder.get_branch()
2536
        b.lock_read()
2537
        self.addCleanup(b.unlock)
2538
        vf = b.repository.inventories
2539
        return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2540
2541
    def test_get_empty(self):
2542
        vf = self.get_ordering_vf({})
2543
        self.assertEqual([], vf.calls)
2544
2545
    def test_get_record_stream_topological(self):
2546
        vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2547
        request_keys = [('B',), ('C',), ('D',), ('A',)]
2548
        keys = [r.key for r in vf.get_record_stream(request_keys,
2549
                                    'topological', False)]
2550
        # We should have gotten the keys in topological order
2551
        self.assertEqual([('A',), ('B',), ('C',), ('D',)], keys)
2552
        # And recorded that the request was made
2553
        self.assertEqual([('get_record_stream', request_keys, 'topological',
2554
                           False)], vf.calls)
2555
2556
    def test_get_record_stream_ordered(self):
2557
        vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2558
        request_keys = [('B',), ('C',), ('D',), ('A',)]
2559
        keys = [r.key for r in vf.get_record_stream(request_keys,
2560
                                   'unordered', False)]
2561
        # They should be returned based on their priority
2562
        self.assertEqual([('D',), ('B',), ('A',), ('C',)], keys)
2563
        # And the request recorded
2564
        self.assertEqual([('get_record_stream', request_keys, 'unordered',
2565
                           False)], vf.calls)
2566
2567
    def test_get_record_stream_implicit_order(self):
2568
        vf = self.get_ordering_vf({('B',): 2, ('D',): 1})
2569
        request_keys = [('B',), ('C',), ('D',), ('A',)]
2570
        keys = [r.key for r in vf.get_record_stream(request_keys,
2571
                                   'unordered', False)]
2572
        # A and C are not in the map, so they get sorted to the front. A comes
2573
        # before C alphabetically, so it comes back first
2574
        self.assertEqual([('A',), ('C',), ('D',), ('B',)], keys)
2575
        # And the request recorded
2576
        self.assertEqual([('get_record_stream', request_keys, 'unordered',
2577
                           False)], vf.calls)