/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/tests/per_versionedfile.py

  • Committer: Jelmer Vernooij
  • Date: 2019-05-11 18:38:21 UTC
  • mto: This revision was merged to the branch mainline in revision 7301.
  • Revision ID: jelmer@jelmer.uk-20190511183821-3xl6jo6kiy5mf4sa
Integrate _urllib2_wrappers.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/tests/per_versionedfile.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
2
2
#
3
3
# Authors:
4
4
#   Johan Rydberg <jrydberg@gnu.org>
21
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
22
# considered typical and check that it can be detected/corrected.
23
23
 
24
 
from itertools import chain, izip
25
 
from StringIO import StringIO
 
24
from gzip import GzipFile
 
25
import itertools
26
26
 
27
 
from bzrlib import (
 
27
from .. import (
28
28
    errors,
29
29
    graph as _mod_graph,
30
 
    groupcompress,
31
 
    knit as _mod_knit,
32
30
    osutils,
33
31
    progress,
 
32
    transport,
34
33
    ui,
35
34
    )
36
 
from bzrlib.errors import (
37
 
                           RevisionNotPresent,
38
 
                           RevisionAlreadyPresent,
39
 
                           WeaveParentMismatch
40
 
                           )
41
 
from bzrlib.knit import (
 
35
from ..bzr import (
 
36
    groupcompress,
 
37
    knit as _mod_knit,
 
38
    )
 
39
from ..errors import (
 
40
    RevisionNotPresent,
 
41
    RevisionAlreadyPresent,
 
42
    )
 
43
from ..bzr.knit import (
42
44
    cleanup_pack_knit,
43
45
    make_file_factory,
44
46
    make_pack_factory,
45
 
    KnitAnnotateFactory,
46
 
    KnitPlainFactory,
47
 
    )
48
 
from bzrlib.tests import (
 
47
    )
 
48
from ..sixish import (
 
49
    BytesIO,
 
50
    zip,
 
51
    )
 
52
from . import (
49
53
    TestCase,
50
54
    TestCaseWithMemoryTransport,
51
55
    TestNotApplicable,
52
56
    TestSkipped,
53
 
    condition_isinstance,
54
 
    split_suite_by_condition,
55
 
    multiply_tests,
56
57
    )
57
 
from bzrlib.tests.http_utils import TestCaseWithWebserver
58
 
from bzrlib.trace import mutter
59
 
from bzrlib.transport import get_transport
60
 
from bzrlib.transport.memory import MemoryTransport
61
 
from bzrlib.tsort import topo_sort
62
 
from bzrlib.tuned_gzip import GzipFile
63
 
import bzrlib.versionedfile as versionedfile
64
 
from bzrlib.versionedfile import (
 
58
from .http_utils import TestCaseWithWebserver
 
59
from ..transport.memory import MemoryTransport
 
60
from ..bzr import versionedfile as versionedfile
 
61
from ..bzr.versionedfile import (
65
62
    ConstantMapper,
66
63
    HashEscapedPrefixMapper,
67
64
    PrefixMapper,
68
65
    VirtualVersionedFiles,
69
66
    make_versioned_files_factory,
70
67
    )
71
 
from bzrlib.weave import WeaveFile
72
 
from bzrlib.weavefile import read_weave, write_weave
73
 
 
74
 
 
75
 
def load_tests(standard_tests, module, loader):
76
 
    """Parameterize VersionedFiles tests for different implementations."""
77
 
    to_adapt, result = split_suite_by_condition(
78
 
        standard_tests, condition_isinstance(TestVersionedFiles))
79
 
    # We want to be sure of behaviour for:
80
 
    # weaves prefix layout (weave texts)
81
 
    # individually named weaves (weave inventories)
82
 
    # annotated knits - prefix|hash|hash-escape layout, we test the third only
83
 
    #                   as it is the most complex mapper.
84
 
    # individually named knits
85
 
    # individual no-graph knits in packs (signatures)
86
 
    # individual graph knits in packs (inventories)
87
 
    # individual graph nocompression knits in packs (revisions)
88
 
    # plain text knits in packs (texts)
89
 
    len_one_scenarios = [
90
 
        ('weave-named', {
91
 
            'cleanup':None,
92
 
            'factory':make_versioned_files_factory(WeaveFile,
93
 
                ConstantMapper('inventory')),
94
 
            'graph':True,
95
 
            'key_length':1,
96
 
            'support_partial_insertion': False,
97
 
            }),
98
 
        ('named-knit', {
99
 
            'cleanup':None,
100
 
            'factory':make_file_factory(False, ConstantMapper('revisions')),
101
 
            'graph':True,
102
 
            'key_length':1,
103
 
            'support_partial_insertion': False,
104
 
            }),
105
 
        ('named-nograph-nodelta-knit-pack', {
106
 
            'cleanup':cleanup_pack_knit,
107
 
            'factory':make_pack_factory(False, False, 1),
108
 
            'graph':False,
109
 
            'key_length':1,
110
 
            'support_partial_insertion': False,
111
 
            }),
112
 
        ('named-graph-knit-pack', {
113
 
            'cleanup':cleanup_pack_knit,
114
 
            'factory':make_pack_factory(True, True, 1),
115
 
            'graph':True,
116
 
            'key_length':1,
117
 
            'support_partial_insertion': True,
118
 
            }),
119
 
        ('named-graph-nodelta-knit-pack', {
120
 
            'cleanup':cleanup_pack_knit,
121
 
            'factory':make_pack_factory(True, False, 1),
122
 
            'graph':True,
123
 
            'key_length':1,
124
 
            'support_partial_insertion': False,
125
 
            }),
126
 
        ('groupcompress-nograph', {
127
 
            'cleanup':groupcompress.cleanup_pack_group,
128
 
            'factory':groupcompress.make_pack_factory(False, False, 1),
129
 
            'graph': False,
130
 
            'key_length':1,
131
 
            'support_partial_insertion':False,
132
 
            }),
133
 
        ]
134
 
    len_two_scenarios = [
135
 
        ('weave-prefix', {
136
 
            'cleanup':None,
137
 
            'factory':make_versioned_files_factory(WeaveFile,
138
 
                PrefixMapper()),
139
 
            'graph':True,
140
 
            'key_length':2,
141
 
            'support_partial_insertion': False,
142
 
            }),
143
 
        ('annotated-knit-escape', {
144
 
            'cleanup':None,
145
 
            'factory':make_file_factory(True, HashEscapedPrefixMapper()),
146
 
            'graph':True,
147
 
            'key_length':2,
148
 
            'support_partial_insertion': False,
149
 
            }),
150
 
        ('plain-knit-pack', {
151
 
            'cleanup':cleanup_pack_knit,
152
 
            'factory':make_pack_factory(True, True, 2),
153
 
            'graph':True,
154
 
            'key_length':2,
155
 
            'support_partial_insertion': True,
156
 
            }),
157
 
        ('groupcompress', {
158
 
            'cleanup':groupcompress.cleanup_pack_group,
159
 
            'factory':groupcompress.make_pack_factory(True, False, 1),
160
 
            'graph': True,
161
 
            'key_length':1,
162
 
            'support_partial_insertion':False,
163
 
            }),
164
 
        ]
165
 
    scenarios = len_one_scenarios + len_two_scenarios
166
 
    return multiply_tests(to_adapt, scenarios, result)
 
68
from ..bzr.weave import (
 
69
    WeaveFile,
 
70
    WeaveInvalidChecksum,
 
71
    )
 
72
from ..bzr.weavefile import write_weave
 
73
from .scenarios import load_tests_apply_scenarios
 
74
 
 
75
 
 
76
load_tests = load_tests_apply_scenarios
167
77
 
168
78
 
169
79
def get_diamond_vf(f, trailing_eol=True, left_only=False):
172
82
    :param trailing_eol: If True end the last line with \n.
173
83
    """
174
84
    parents = {
175
 
        'origin': (),
176
 
        'base': (('origin',),),
177
 
        'left': (('base',),),
178
 
        'right': (('base',),),
179
 
        'merged': (('left',), ('right',)),
 
85
        b'origin': (),
 
86
        b'base': ((b'origin',),),
 
87
        b'left': ((b'base',),),
 
88
        b'right': ((b'base',),),
 
89
        b'merged': ((b'left',), (b'right',)),
180
90
        }
181
91
    # insert a diamond graph to exercise deltas and merges.
182
92
    if trailing_eol:
183
 
        last_char = '\n'
 
93
        last_char = b'\n'
184
94
    else:
185
 
        last_char = ''
186
 
    f.add_lines('origin', [], ['origin' + last_char])
187
 
    f.add_lines('base', ['origin'], ['base' + last_char])
188
 
    f.add_lines('left', ['base'], ['base\n', 'left' + last_char])
 
95
        last_char = b''
 
96
    f.add_lines(b'origin', [], [b'origin' + last_char])
 
97
    f.add_lines(b'base', [b'origin'], [b'base' + last_char])
 
98
    f.add_lines(b'left', [b'base'], [b'base\n', b'left' + last_char])
189
99
    if not left_only:
190
 
        f.add_lines('right', ['base'],
191
 
            ['base\n', 'right' + last_char])
192
 
        f.add_lines('merged', ['left', 'right'],
193
 
            ['base\n', 'left\n', 'right\n', 'merged' + last_char])
 
100
        f.add_lines(b'right', [b'base'],
 
101
                    [b'base\n', b'right' + last_char])
 
102
        f.add_lines(b'merged', [b'left', b'right'],
 
103
                    [b'base\n', b'left\n', b'right\n', b'merged' + last_char])
194
104
    return f, parents
195
105
 
196
106
 
197
107
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
198
 
    nograph=False, nokeys=False):
 
108
                      nograph=False, nokeys=False):
199
109
    """Get a diamond graph to exercise deltas and merges.
200
110
 
201
111
    This creates a 5-node graph in files. If files supports 2-length keys two
217
127
    if key_length == 1:
218
128
        prefixes = [()]
219
129
    else:
220
 
        prefixes = [('FileA',), ('FileB',)]
 
130
        prefixes = [(b'FileA',), (b'FileB',)]
221
131
    # insert a diamond graph to exercise deltas and merges.
222
132
    if trailing_eol:
223
 
        last_char = '\n'
 
133
        last_char = b'\n'
224
134
    else:
225
 
        last_char = ''
 
135
        last_char = b''
226
136
    result = []
 
137
 
227
138
    def get_parents(suffix_list):
228
139
        if nograph:
229
140
            return ()
230
141
        else:
231
142
            result = [prefix + suffix for suffix in suffix_list]
232
143
            return result
 
144
 
233
145
    def get_key(suffix):
234
146
        if nokeys:
235
147
            return (None, )
238
150
    # we loop over each key because that spreads the inserts across prefixes,
239
151
    # which is how commit operates.
240
152
    for prefix in prefixes:
241
 
        result.append(files.add_lines(prefix + get_key('origin'), (),
242
 
            ['origin' + last_char]))
243
 
    for prefix in prefixes:
244
 
        result.append(files.add_lines(prefix + get_key('base'),
245
 
            get_parents([('origin',)]), ['base' + last_char]))
246
 
    for prefix in prefixes:
247
 
        result.append(files.add_lines(prefix + get_key('left'),
248
 
            get_parents([('base',)]),
249
 
            ['base\n', 'left' + last_char]))
 
153
        result.append(files.add_lines(prefix + get_key(b'origin'), (),
 
154
                                      [b'origin' + last_char]))
 
155
    for prefix in prefixes:
 
156
        result.append(files.add_lines(prefix + get_key(b'base'),
 
157
                                      get_parents([(b'origin',)]), [b'base' + last_char]))
 
158
    for prefix in prefixes:
 
159
        result.append(files.add_lines(prefix + get_key(b'left'),
 
160
                                      get_parents([(b'base',)]),
 
161
                                      [b'base\n', b'left' + last_char]))
250
162
    if not left_only:
251
163
        for prefix in prefixes:
252
 
            result.append(files.add_lines(prefix + get_key('right'),
253
 
                get_parents([('base',)]),
254
 
                ['base\n', 'right' + last_char]))
 
164
            result.append(files.add_lines(prefix + get_key(b'right'),
 
165
                                          get_parents([(b'base',)]),
 
166
                                          [b'base\n', b'right' + last_char]))
255
167
        for prefix in prefixes:
256
 
            result.append(files.add_lines(prefix + get_key('merged'),
257
 
                get_parents([('left',), ('right',)]),
258
 
                ['base\n', 'left\n', 'right\n', 'merged' + last_char]))
 
168
            result.append(files.add_lines(prefix + get_key(b'merged'),
 
169
                                          get_parents(
 
170
                                              [(b'left',), (b'right',)]),
 
171
                                          [b'base\n', b'left\n', b'right\n', b'merged' + last_char]))
259
172
    return result
260
173
 
261
174
 
274
187
 
275
188
    def test_add(self):
276
189
        f = self.get_file()
277
 
        f.add_lines('r0', [], ['a\n', 'b\n'])
278
 
        f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
 
190
        f.add_lines(b'r0', [], [b'a\n', b'b\n'])
 
191
        f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
 
192
 
279
193
        def verify_file(f):
280
194
            versions = f.versions()
281
 
            self.assertTrue('r0' in versions)
282
 
            self.assertTrue('r1' in versions)
283
 
            self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
284
 
            self.assertEquals(f.get_text('r0'), 'a\nb\n')
285
 
            self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
 
195
            self.assertTrue(b'r0' in versions)
 
196
            self.assertTrue(b'r1' in versions)
 
197
            self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
 
198
            self.assertEqual(f.get_text(b'r0'), b'a\nb\n')
 
199
            self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
286
200
            self.assertEqual(2, len(f))
287
201
            self.assertEqual(2, f.num_versions())
288
202
 
289
203
            self.assertRaises(RevisionNotPresent,
290
 
                f.add_lines, 'r2', ['foo'], [])
 
204
                              f.add_lines, b'r2', [b'foo'], [])
291
205
            self.assertRaises(RevisionAlreadyPresent,
292
 
                f.add_lines, 'r1', [], [])
 
206
                              f.add_lines, b'r1', [], [])
293
207
        verify_file(f)
294
208
        # this checks that reopen with create=True does not break anything.
295
209
        f = self.reopen_file(create=True)
298
212
    def test_adds_with_parent_texts(self):
299
213
        f = self.get_file()
300
214
        parent_texts = {}
301
 
        _, _, parent_texts['r0'] = f.add_lines('r0', [], ['a\n', 'b\n'])
 
215
        _, _, parent_texts[b'r0'] = f.add_lines(b'r0', [], [b'a\n', b'b\n'])
302
216
        try:
303
 
            _, _, parent_texts['r1'] = f.add_lines_with_ghosts('r1',
304
 
                ['r0', 'ghost'], ['b\n', 'c\n'], parent_texts=parent_texts)
 
217
            _, _, parent_texts[b'r1'] = f.add_lines_with_ghosts(b'r1',
 
218
                                                                [b'r0', b'ghost'], [b'b\n', b'c\n'], parent_texts=parent_texts)
305
219
        except NotImplementedError:
306
220
            # if the format doesn't support ghosts, just add normally.
307
 
            _, _, parent_texts['r1'] = f.add_lines('r1',
308
 
                ['r0'], ['b\n', 'c\n'], parent_texts=parent_texts)
309
 
        f.add_lines('r2', ['r1'], ['c\n', 'd\n'], parent_texts=parent_texts)
310
 
        self.assertNotEqual(None, parent_texts['r0'])
311
 
        self.assertNotEqual(None, parent_texts['r1'])
 
221
            _, _, parent_texts[b'r1'] = f.add_lines(b'r1',
 
222
                                                    [b'r0'], [b'b\n', b'c\n'], parent_texts=parent_texts)
 
223
        f.add_lines(b'r2', [b'r1'], [b'c\n', b'd\n'],
 
224
                    parent_texts=parent_texts)
 
225
        self.assertNotEqual(None, parent_texts[b'r0'])
 
226
        self.assertNotEqual(None, parent_texts[b'r1'])
 
227
 
312
228
        def verify_file(f):
313
229
            versions = f.versions()
314
 
            self.assertTrue('r0' in versions)
315
 
            self.assertTrue('r1' in versions)
316
 
            self.assertTrue('r2' in versions)
317
 
            self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
318
 
            self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
319
 
            self.assertEquals(f.get_lines('r2'), ['c\n', 'd\n'])
 
230
            self.assertTrue(b'r0' in versions)
 
231
            self.assertTrue(b'r1' in versions)
 
232
            self.assertTrue(b'r2' in versions)
 
233
            self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
 
234
            self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
 
235
            self.assertEqual(f.get_lines(b'r2'), [b'c\n', b'd\n'])
320
236
            self.assertEqual(3, f.num_versions())
321
 
            origins = f.annotate('r1')
322
 
            self.assertEquals(origins[0][0], 'r0')
323
 
            self.assertEquals(origins[1][0], 'r1')
324
 
            origins = f.annotate('r2')
325
 
            self.assertEquals(origins[0][0], 'r1')
326
 
            self.assertEquals(origins[1][0], 'r2')
 
237
            origins = f.annotate(b'r1')
 
238
            self.assertEqual(origins[0][0], b'r0')
 
239
            self.assertEqual(origins[1][0], b'r1')
 
240
            origins = f.annotate(b'r2')
 
241
            self.assertEqual(origins[0][0], b'r1')
 
242
            self.assertEqual(origins[1][0], b'r2')
327
243
 
328
244
        verify_file(f)
329
245
        f = self.reopen_file()
334
250
        # versioned files version sequences of bytes only.
335
251
        vf = self.get_file()
336
252
        self.assertRaises(errors.BzrBadParameterUnicode,
337
 
            vf.add_lines, 'a', [], ['a\n', u'b\n', 'c\n'])
 
253
                          vf.add_lines, b'a', [], [b'a\n', u'b\n', b'c\n'])
338
254
        self.assertRaises(
339
255
            (errors.BzrBadParameterUnicode, NotImplementedError),
340
 
            vf.add_lines_with_ghosts, 'a', [], ['a\n', u'b\n', 'c\n'])
 
256
            vf.add_lines_with_ghosts, b'a', [], [b'a\n', u'b\n', b'c\n'])
341
257
 
342
258
    def test_add_follows_left_matching_blocks(self):
343
259
        """If we change left_matching_blocks, delta changes
348
264
        vf = self.get_file()
349
265
        if isinstance(vf, WeaveFile):
350
266
            raise TestSkipped("WeaveFile ignores left_matching_blocks")
351
 
        vf.add_lines('1', [], ['a\n'])
352
 
        vf.add_lines('2', ['1'], ['a\n', 'a\n', 'a\n'],
 
267
        vf.add_lines(b'1', [], [b'a\n'])
 
268
        vf.add_lines(b'2', [b'1'], [b'a\n', b'a\n', b'a\n'],
353
269
                     left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
354
 
        self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('2'))
355
 
        vf.add_lines('3', ['1'], ['a\n', 'a\n', 'a\n'],
 
270
        self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'2'))
 
271
        vf.add_lines(b'3', [b'1'], [b'a\n', b'a\n', b'a\n'],
356
272
                     left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
357
 
        self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('3'))
 
273
        self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'3'))
358
274
 
359
275
    def test_inline_newline_throws(self):
360
276
        # \r characters are not permitted in lines being added
361
277
        vf = self.get_file()
362
278
        self.assertRaises(errors.BzrBadParameterContainsNewline,
363
 
            vf.add_lines, 'a', [], ['a\n\n'])
 
279
                          vf.add_lines, b'a', [], [b'a\n\n'])
364
280
        self.assertRaises(
365
281
            (errors.BzrBadParameterContainsNewline, NotImplementedError),
366
 
            vf.add_lines_with_ghosts, 'a', [], ['a\n\n'])
 
282
            vf.add_lines_with_ghosts, b'a', [], [b'a\n\n'])
367
283
        # but inline CR's are allowed
368
 
        vf.add_lines('a', [], ['a\r\n'])
 
284
        vf.add_lines(b'a', [], [b'a\r\n'])
369
285
        try:
370
 
            vf.add_lines_with_ghosts('b', [], ['a\r\n'])
 
286
            vf.add_lines_with_ghosts(b'b', [], [b'a\r\n'])
371
287
        except NotImplementedError:
372
288
            pass
373
289
 
374
290
    def test_add_reserved(self):
375
291
        vf = self.get_file()
376
292
        self.assertRaises(errors.ReservedId,
377
 
            vf.add_lines, 'a:', [], ['a\n', 'b\n', 'c\n'])
 
293
                          vf.add_lines, b'a:', [], [b'a\n', b'b\n', b'c\n'])
378
294
 
379
295
    def test_add_lines_nostoresha(self):
380
296
        """When nostore_sha is supplied using old content raises."""
381
297
        vf = self.get_file()
382
 
        empty_text = ('a', [])
383
 
        sample_text_nl = ('b', ["foo\n", "bar\n"])
384
 
        sample_text_no_nl = ('c', ["foo\n", "bar"])
 
298
        empty_text = (b'a', [])
 
299
        sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
 
300
        sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
385
301
        shas = []
386
302
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
387
303
            sha, _, _ = vf.add_lines(version, [], lines)
388
304
            shas.append(sha)
389
305
        # we now have a copy of all the lines in the vf.
390
306
        for sha, (version, lines) in zip(
391
 
            shas, (empty_text, sample_text_nl, sample_text_no_nl)):
 
307
                shas, (empty_text, sample_text_nl, sample_text_no_nl)):
392
308
            self.assertRaises(errors.ExistingContent,
393
 
                vf.add_lines, version + "2", [], lines,
394
 
                nostore_sha=sha)
 
309
                              vf.add_lines, version + b"2", [], lines,
 
310
                              nostore_sha=sha)
395
311
            # and no new version should have been added.
396
312
            self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
397
 
                version + "2")
 
313
                              version + b"2")
398
314
 
399
315
    def test_add_lines_with_ghosts_nostoresha(self):
400
316
        """When nostore_sha is supplied using old content raises."""
401
317
        vf = self.get_file()
402
 
        empty_text = ('a', [])
403
 
        sample_text_nl = ('b', ["foo\n", "bar\n"])
404
 
        sample_text_no_nl = ('c', ["foo\n", "bar"])
 
318
        empty_text = (b'a', [])
 
319
        sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
 
320
        sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
405
321
        shas = []
406
322
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
407
323
            sha, _, _ = vf.add_lines(version, [], lines)
409
325
        # we now have a copy of all the lines in the vf.
410
326
        # is the test applicable to this vf implementation?
411
327
        try:
412
 
            vf.add_lines_with_ghosts('d', [], [])
 
328
            vf.add_lines_with_ghosts(b'd', [], [])
413
329
        except NotImplementedError:
414
330
            raise TestSkipped("add_lines_with_ghosts is optional")
415
331
        for sha, (version, lines) in zip(
416
 
            shas, (empty_text, sample_text_nl, sample_text_no_nl)):
 
332
                shas, (empty_text, sample_text_nl, sample_text_no_nl)):
417
333
            self.assertRaises(errors.ExistingContent,
418
 
                vf.add_lines_with_ghosts, version + "2", [], lines,
419
 
                nostore_sha=sha)
 
334
                              vf.add_lines_with_ghosts, version + b"2", [], lines,
 
335
                              nostore_sha=sha)
420
336
            # and no new version should have been added.
421
337
            self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
422
 
                version + "2")
 
338
                              version + b"2")
423
339
 
424
340
    def test_add_lines_return_value(self):
425
341
        # add_lines should return the sha1 and the text size.
426
342
        vf = self.get_file()
427
 
        empty_text = ('a', [])
428
 
        sample_text_nl = ('b', ["foo\n", "bar\n"])
429
 
        sample_text_no_nl = ('c', ["foo\n", "bar"])
 
343
        empty_text = (b'a', [])
 
344
        sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
 
345
        sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
430
346
        # check results for the three cases:
431
347
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
432
348
            # the first two elements are the same for all versioned files:
435
351
            result = vf.add_lines(version, [], lines)
436
352
            self.assertEqual(3, len(result))
437
353
            self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
438
 
                result[0:2])
 
354
                             result[0:2])
439
355
        # parents should not affect the result:
440
356
        lines = sample_text_nl[1]
441
357
        self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
442
 
            vf.add_lines('d', ['b', 'c'], lines)[0:2])
 
358
                         vf.add_lines(b'd', [b'b', b'c'], lines)[0:2])
443
359
 
444
360
    def test_get_reserved(self):
445
361
        vf = self.get_file()
446
 
        self.assertRaises(errors.ReservedId, vf.get_texts, ['b:'])
447
 
        self.assertRaises(errors.ReservedId, vf.get_lines, 'b:')
448
 
        self.assertRaises(errors.ReservedId, vf.get_text, 'b:')
 
362
        self.assertRaises(errors.ReservedId, vf.get_texts, [b'b:'])
 
363
        self.assertRaises(errors.ReservedId, vf.get_lines, b'b:')
 
364
        self.assertRaises(errors.ReservedId, vf.get_text, b'b:')
449
365
 
450
366
    def test_add_unchanged_last_line_noeol_snapshot(self):
451
367
        """Add a text with an unchanged last line with no eol should work."""
460
376
        for length in range(20):
461
377
            version_lines = {}
462
378
            vf = self.get_file('case-%d' % length)
463
 
            prefix = 'step-%d'
 
379
            prefix = b'step-%d'
464
380
            parents = []
465
381
            for step in range(length):
466
382
                version = prefix % step
467
 
                lines = (['prelude \n'] * step) + ['line']
 
383
                lines = ([b'prelude \n'] * step) + [b'line']
468
384
                vf.add_lines(version, parents, lines)
469
385
                version_lines[version] = lines
470
386
                parents = [version]
471
 
            vf.add_lines('no-eol', parents, ['line'])
 
387
            vf.add_lines(b'no-eol', parents, [b'line'])
472
388
            vf.get_texts(version_lines.keys())
473
 
            self.assertEqualDiff('line', vf.get_text('no-eol'))
 
389
            self.assertEqualDiff(b'line', vf.get_text(b'no-eol'))
474
390
 
475
391
    def test_get_texts_eol_variation(self):
476
392
        # similar to the failure in <http://bugs.launchpad.net/234748>
477
393
        vf = self.get_file()
478
 
        sample_text_nl = ["line\n"]
479
 
        sample_text_no_nl = ["line"]
 
394
        sample_text_nl = [b"line\n"]
 
395
        sample_text_no_nl = [b"line"]
480
396
        versions = []
481
397
        version_lines = {}
482
398
        parents = []
483
399
        for i in range(4):
484
 
            version = 'v%d' % i
 
400
            version = b'v%d' % i
485
401
            if i % 2:
486
402
                lines = sample_text_nl
487
403
            else:
493
409
            # (which is what this test tests) will generate a correct line
494
410
            # delta (which is to say, an empty delta).
495
411
            vf.add_lines(version, parents, lines,
496
 
                left_matching_blocks=[(0, 0, 1)])
 
412
                         left_matching_blocks=[(0, 0, 1)])
497
413
            parents = [version]
498
414
            versions.append(version)
499
415
            version_lines[version] = lines
503
419
 
504
420
    def test_add_lines_with_matching_blocks_noeol_last_line(self):
505
421
        """Add a text with an unchanged last line with no eol should work."""
506
 
        from bzrlib import multiparent
 
422
        from breezy import multiparent
507
423
        # Hand verified sha1 of the text we're adding.
508
424
        sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
509
425
        # Create a mpdiff which adds a new line before the trailing line, and
511
427
        # Test adding this in two situations:
512
428
        # On top of a new insertion
513
429
        vf = self.get_file('fulltext')
514
 
        vf.add_lines('noeol', [], ['line'])
515
 
        vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
516
 
            left_matching_blocks=[(0, 1, 1)])
517
 
        self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
 
430
        vf.add_lines(b'noeol', [], [b'line'])
 
431
        vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
 
432
                     left_matching_blocks=[(0, 1, 1)])
 
433
        self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
518
434
        # On top of a delta
519
435
        vf = self.get_file('delta')
520
 
        vf.add_lines('base', [], ['line'])
521
 
        vf.add_lines('noeol', ['base'], ['prelude\n', 'line'])
522
 
        vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
523
 
            left_matching_blocks=[(1, 1, 1)])
524
 
        self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
 
436
        vf.add_lines(b'base', [], [b'line'])
 
437
        vf.add_lines(b'noeol', [b'base'], [b'prelude\n', b'line'])
 
438
        vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
 
439
                     left_matching_blocks=[(1, 1, 1)])
 
440
        self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
525
441
 
526
442
    def test_make_mpdiffs(self):
527
 
        from bzrlib import multiparent
 
443
        from breezy import multiparent
528
444
        vf = self.get_file('foo')
529
445
        sha1s = self._setup_for_deltas(vf)
530
446
        new_vf = self.get_file('bar')
538
454
    def test_make_mpdiffs_with_ghosts(self):
539
455
        vf = self.get_file('foo')
540
456
        try:
541
 
            vf.add_lines_with_ghosts('text', ['ghost'], ['line\n'])
 
457
            vf.add_lines_with_ghosts(b'text', [b'ghost'], [b'line\n'])
542
458
        except NotImplementedError:
543
459
            # old Weave formats do not allow ghosts
544
460
            return
545
 
        self.assertRaises(errors.RevisionNotPresent, vf.make_mpdiffs, ['ghost'])
 
461
        self.assertRaises(errors.RevisionNotPresent,
 
462
                          vf.make_mpdiffs, [b'ghost'])
546
463
 
547
464
    def _setup_for_deltas(self, f):
548
465
        self.assertFalse(f.has_version('base'))
549
466
        # add texts that should trip the knit maximum delta chain threshold
550
467
        # as well as doing parallel chains of data in knits.
551
468
        # this is done by two chains of 25 insertions
552
 
        f.add_lines('base', [], ['line\n'])
553
 
        f.add_lines('noeol', ['base'], ['line'])
 
469
        f.add_lines(b'base', [], [b'line\n'])
 
470
        f.add_lines(b'noeol', [b'base'], [b'line'])
554
471
        # detailed eol tests:
555
472
        # shared last line with parent no-eol
556
 
        f.add_lines('noeolsecond', ['noeol'], ['line\n', 'line'])
 
473
        f.add_lines(b'noeolsecond', [b'noeol'], [b'line\n', b'line'])
557
474
        # differing last line with parent, both no-eol
558
 
        f.add_lines('noeolnotshared', ['noeolsecond'], ['line\n', 'phone'])
 
475
        f.add_lines(b'noeolnotshared', [b'noeolsecond'], [b'line\n', b'phone'])
559
476
        # add eol following a noneol parent, change content
560
 
        f.add_lines('eol', ['noeol'], ['phone\n'])
 
477
        f.add_lines(b'eol', [b'noeol'], [b'phone\n'])
561
478
        # add eol following a noneol parent, no change content
562
 
        f.add_lines('eolline', ['noeol'], ['line\n'])
 
479
        f.add_lines(b'eolline', [b'noeol'], [b'line\n'])
563
480
        # noeol with no parents:
564
 
        f.add_lines('noeolbase', [], ['line'])
 
481
        f.add_lines(b'noeolbase', [], [b'line'])
565
482
        # noeol preceeding its leftmost parent in the output:
566
483
        # this is done by making it a merge of two parents with no common
567
484
        # anestry: noeolbase and noeol with the
568
485
        # later-inserted parent the leftmost.
569
 
        f.add_lines('eolbeforefirstparent', ['noeolbase', 'noeol'], ['line'])
 
486
        f.add_lines(b'eolbeforefirstparent', [
 
487
                    b'noeolbase', b'noeol'], [b'line'])
570
488
        # two identical eol texts
571
 
        f.add_lines('noeoldup', ['noeol'], ['line'])
572
 
        next_parent = 'base'
573
 
        text_name = 'chain1-'
574
 
        text = ['line\n']
575
 
        sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
576
 
                 1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
577
 
                 2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
578
 
                 3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
579
 
                 4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
580
 
                 5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
581
 
                 6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
582
 
                 7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
583
 
                 8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
584
 
                 9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
585
 
                 10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
586
 
                 11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
587
 
                 12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
588
 
                 13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
589
 
                 14:'2c4b1736566b8ca6051e668de68650686a3922f2',
590
 
                 15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
591
 
                 16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
592
 
                 17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
593
 
                 18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
594
 
                 19:'1ebed371807ba5935958ad0884595126e8c4e823',
595
 
                 20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
596
 
                 21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
597
 
                 22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
598
 
                 23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
599
 
                 24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
600
 
                 25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
 
489
        f.add_lines(b'noeoldup', [b'noeol'], [b'line'])
 
490
        next_parent = b'base'
 
491
        text_name = b'chain1-'
 
492
        text = [b'line\n']
 
493
        sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
 
494
                 1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
 
495
                 2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
 
496
                 3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
 
497
                 4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
 
498
                 5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
 
499
                 6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
 
500
                 7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
 
501
                 8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
 
502
                 9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
 
503
                 10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
 
504
                 11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
 
505
                 12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
 
506
                 13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
 
507
                 14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
 
508
                 15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
 
509
                 16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
 
510
                 17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
 
511
                 18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
 
512
                 19: b'1ebed371807ba5935958ad0884595126e8c4e823',
 
513
                 20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
 
514
                 21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
 
515
                 22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
 
516
                 23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
 
517
                 24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
 
518
                 25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
601
519
                 }
602
520
        for depth in range(26):
603
 
            new_version = text_name + '%s' % depth
604
 
            text = text + ['line\n']
 
521
            new_version = text_name + b'%d' % depth
 
522
            text = text + [b'line\n']
605
523
            f.add_lines(new_version, [next_parent], text)
606
524
            next_parent = new_version
607
 
        next_parent = 'base'
608
 
        text_name = 'chain2-'
609
 
        text = ['line\n']
 
525
        next_parent = b'base'
 
526
        text_name = b'chain2-'
 
527
        text = [b'line\n']
610
528
        for depth in range(26):
611
 
            new_version = text_name + '%s' % depth
612
 
            text = text + ['line\n']
 
529
            new_version = text_name + b'%d' % depth
 
530
            text = text + [b'line\n']
613
531
            f.add_lines(new_version, [next_parent], text)
614
532
            next_parent = new_version
615
533
        return sha1s
617
535
    def test_ancestry(self):
618
536
        f = self.get_file()
619
537
        self.assertEqual([], f.get_ancestry([]))
620
 
        f.add_lines('r0', [], ['a\n', 'b\n'])
621
 
        f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
622
 
        f.add_lines('r2', ['r0'], ['b\n', 'c\n'])
623
 
        f.add_lines('r3', ['r2'], ['b\n', 'c\n'])
624
 
        f.add_lines('rM', ['r1', 'r2'], ['b\n', 'c\n'])
 
538
        f.add_lines(b'r0', [], [b'a\n', b'b\n'])
 
539
        f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
 
540
        f.add_lines(b'r2', [b'r0'], [b'b\n', b'c\n'])
 
541
        f.add_lines(b'r3', [b'r2'], [b'b\n', b'c\n'])
 
542
        f.add_lines(b'rM', [b'r1', b'r2'], [b'b\n', b'c\n'])
625
543
        self.assertEqual([], f.get_ancestry([]))
626
 
        versions = f.get_ancestry(['rM'])
 
544
        versions = f.get_ancestry([b'rM'])
627
545
        # there are some possibilities:
628
546
        # r0 r1 r2 rM r3
629
547
        # r0 r1 r2 r3 rM
630
548
        # etc
631
549
        # so we check indexes
632
 
        r0 = versions.index('r0')
633
 
        r1 = versions.index('r1')
634
 
        r2 = versions.index('r2')
635
 
        self.assertFalse('r3' in versions)
636
 
        rM = versions.index('rM')
 
550
        r0 = versions.index(b'r0')
 
551
        r1 = versions.index(b'r1')
 
552
        r2 = versions.index(b'r2')
 
553
        self.assertFalse(b'r3' in versions)
 
554
        rM = versions.index(b'rM')
637
555
        self.assertTrue(r0 < r1)
638
556
        self.assertTrue(r0 < r2)
639
557
        self.assertTrue(r1 < rM)
640
558
        self.assertTrue(r2 < rM)
641
559
 
642
560
        self.assertRaises(RevisionNotPresent,
643
 
            f.get_ancestry, ['rM', 'rX'])
 
561
                          f.get_ancestry, [b'rM', b'rX'])
644
562
 
645
 
        self.assertEqual(set(f.get_ancestry('rM')),
646
 
            set(f.get_ancestry('rM', topo_sorted=False)))
 
563
        self.assertEqual(set(f.get_ancestry(b'rM')),
 
564
                         set(f.get_ancestry(b'rM', topo_sorted=False)))
647
565
 
648
566
    def test_mutate_after_finish(self):
649
567
        self._transaction = 'before'
650
568
        f = self.get_file()
651
569
        self._transaction = 'after'
652
 
        self.assertRaises(errors.OutSideTransaction, f.add_lines, '', [], [])
653
 
        self.assertRaises(errors.OutSideTransaction, f.add_lines_with_ghosts, '', [], [])
 
570
        self.assertRaises(errors.OutSideTransaction, f.add_lines, b'', [], [])
 
571
        self.assertRaises(errors.OutSideTransaction,
 
572
                          f.add_lines_with_ghosts, b'', [], [])
654
573
 
655
574
    def test_copy_to(self):
656
575
        f = self.get_file()
657
 
        f.add_lines('0', [], ['a\n'])
 
576
        f.add_lines(b'0', [], [b'a\n'])
658
577
        t = MemoryTransport()
659
578
        f.copy_to('foo', t)
660
579
        for suffix in self.get_factory().get_suffixes():
667
586
 
668
587
    def test_get_parent_map(self):
669
588
        f = self.get_file()
670
 
        f.add_lines('r0', [], ['a\n', 'b\n'])
671
 
        self.assertEqual(
672
 
            {'r0':()}, f.get_parent_map(['r0']))
673
 
        f.add_lines('r1', ['r0'], ['a\n', 'b\n'])
674
 
        self.assertEqual(
675
 
            {'r1':('r0',)}, f.get_parent_map(['r1']))
676
 
        self.assertEqual(
677
 
            {'r0':(),
678
 
             'r1':('r0',)},
679
 
            f.get_parent_map(['r0', 'r1']))
680
 
        f.add_lines('r2', [], ['a\n', 'b\n'])
681
 
        f.add_lines('r3', [], ['a\n', 'b\n'])
682
 
        f.add_lines('m', ['r0', 'r1', 'r2', 'r3'], ['a\n', 'b\n'])
683
 
        self.assertEqual(
684
 
            {'m':('r0', 'r1', 'r2', 'r3')}, f.get_parent_map(['m']))
685
 
        self.assertEqual({}, f.get_parent_map('y'))
686
 
        self.assertEqual(
687
 
            {'r0':(),
688
 
             'r1':('r0',)},
689
 
            f.get_parent_map(['r0', 'y', 'r1']))
 
589
        f.add_lines(b'r0', [], [b'a\n', b'b\n'])
 
590
        self.assertEqual(
 
591
            {b'r0': ()}, f.get_parent_map([b'r0']))
 
592
        f.add_lines(b'r1', [b'r0'], [b'a\n', b'b\n'])
 
593
        self.assertEqual(
 
594
            {b'r1': (b'r0',)}, f.get_parent_map([b'r1']))
 
595
        self.assertEqual(
 
596
            {b'r0': (),
 
597
             b'r1': (b'r0',)},
 
598
            f.get_parent_map([b'r0', b'r1']))
 
599
        f.add_lines(b'r2', [], [b'a\n', b'b\n'])
 
600
        f.add_lines(b'r3', [], [b'a\n', b'b\n'])
 
601
        f.add_lines(b'm', [b'r0', b'r1', b'r2', b'r3'], [b'a\n', b'b\n'])
 
602
        self.assertEqual(
 
603
            {b'm': (b'r0', b'r1', b'r2', b'r3')}, f.get_parent_map([b'm']))
 
604
        self.assertEqual({}, f.get_parent_map(b'y'))
 
605
        self.assertEqual(
 
606
            {b'r0': (),
 
607
             b'r1': (b'r0',)},
 
608
            f.get_parent_map([b'r0', b'y', b'r1']))
690
609
 
691
610
    def test_annotate(self):
692
611
        f = self.get_file()
693
 
        f.add_lines('r0', [], ['a\n', 'b\n'])
694
 
        f.add_lines('r1', ['r0'], ['c\n', 'b\n'])
695
 
        origins = f.annotate('r1')
696
 
        self.assertEquals(origins[0][0], 'r1')
697
 
        self.assertEquals(origins[1][0], 'r0')
 
612
        f.add_lines(b'r0', [], [b'a\n', b'b\n'])
 
613
        f.add_lines(b'r1', [b'r0'], [b'c\n', b'b\n'])
 
614
        origins = f.annotate(b'r1')
 
615
        self.assertEqual(origins[0][0], b'r1')
 
616
        self.assertEqual(origins[1][0], b'r0')
698
617
 
699
618
        self.assertRaises(RevisionNotPresent,
700
 
            f.annotate, 'foo')
 
619
                          f.annotate, b'foo')
701
620
 
702
621
    def test_detection(self):
703
622
        # Test weaves detect corruption.
708
627
 
709
628
        w = self.get_file_corrupted_text()
710
629
 
711
 
        self.assertEqual('hello\n', w.get_text('v1'))
712
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
713
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
714
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.check)
 
630
        self.assertEqual(b'hello\n', w.get_text(b'v1'))
 
631
        self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
 
632
        self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
 
633
        self.assertRaises(WeaveInvalidChecksum, w.check)
715
634
 
716
635
        w = self.get_file_corrupted_checksum()
717
636
 
718
 
        self.assertEqual('hello\n', w.get_text('v1'))
719
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
720
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
721
 
        self.assertRaises(errors.WeaveInvalidChecksum, w.check)
 
637
        self.assertEqual(b'hello\n', w.get_text(b'v1'))
 
638
        self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
 
639
        self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
 
640
        self.assertRaises(WeaveInvalidChecksum, w.check)
722
641
 
723
642
    def get_file_corrupted_text(self):
724
643
        """Return a versioned file with corrupt text but valid metadata."""
745
664
 
746
665
        vf = self.get_file()
747
666
        # add a base to get included
748
 
        vf.add_lines('base', [], ['base\n'])
 
667
        vf.add_lines(b'base', [], [b'base\n'])
749
668
        # add a ancestor to be included on one side
750
 
        vf.add_lines('lancestor', [], ['lancestor\n'])
 
669
        vf.add_lines(b'lancestor', [], [b'lancestor\n'])
751
670
        # add a ancestor to be included on the other side
752
 
        vf.add_lines('rancestor', ['base'], ['rancestor\n'])
 
671
        vf.add_lines(b'rancestor', [b'base'], [b'rancestor\n'])
753
672
        # add a child of rancestor with no eofile-nl
754
 
        vf.add_lines('child', ['rancestor'], ['base\n', 'child\n'])
 
673
        vf.add_lines(b'child', [b'rancestor'], [b'base\n', b'child\n'])
755
674
        # add a child of lancestor and base to join the two roots
756
 
        vf.add_lines('otherchild',
757
 
                     ['lancestor', 'base'],
758
 
                     ['base\n', 'lancestor\n', 'otherchild\n'])
 
675
        vf.add_lines(b'otherchild',
 
676
                     [b'lancestor', b'base'],
 
677
                     [b'base\n', b'lancestor\n', b'otherchild\n'])
 
678
 
759
679
        def iter_with_versions(versions, expected):
760
680
            # now we need to see what lines are returned, and how often.
761
681
            lines = {}
762
682
            progress = InstrumentedProgress()
763
683
            # iterate over the lines
764
684
            for line in vf.iter_lines_added_or_present_in_versions(versions,
765
 
                pb=progress):
 
685
                                                                   pb=progress):
766
686
                lines.setdefault(line, 0)
767
687
                lines[line] += 1
768
 
            if []!= progress.updates:
 
688
            if [] != progress.updates:
769
689
                self.assertEqual(expected, progress.updates)
770
690
            return lines
771
 
        lines = iter_with_versions(['child', 'otherchild'],
 
691
        lines = iter_with_versions([b'child', b'otherchild'],
772
692
                                   [('Walking content', 0, 2),
773
693
                                    ('Walking content', 1, 2),
774
694
                                    ('Walking content', 2, 2)])
775
695
        # we must see child and otherchild
776
 
        self.assertTrue(lines[('child\n', 'child')] > 0)
777
 
        self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
 
696
        self.assertTrue(lines[(b'child\n', b'child')] > 0)
 
697
        self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
778
698
        # we dont care if we got more than that.
779
699
 
780
700
        # test all lines
785
705
                                          ('Walking content', 4, 5),
786
706
                                          ('Walking content', 5, 5)])
787
707
        # all lines must be seen at least once
788
 
        self.assertTrue(lines[('base\n', 'base')] > 0)
789
 
        self.assertTrue(lines[('lancestor\n', 'lancestor')] > 0)
790
 
        self.assertTrue(lines[('rancestor\n', 'rancestor')] > 0)
791
 
        self.assertTrue(lines[('child\n', 'child')] > 0)
792
 
        self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
 
708
        self.assertTrue(lines[(b'base\n', b'base')] > 0)
 
709
        self.assertTrue(lines[(b'lancestor\n', b'lancestor')] > 0)
 
710
        self.assertTrue(lines[(b'rancestor\n', b'rancestor')] > 0)
 
711
        self.assertTrue(lines[(b'child\n', b'child')] > 0)
 
712
        self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
793
713
 
794
714
    def test_add_lines_with_ghosts(self):
795
715
        # some versioned file formats allow lines to be added with parent
802
722
        parent_id_unicode = u'b\xbfse'
803
723
        parent_id_utf8 = parent_id_unicode.encode('utf8')
804
724
        try:
805
 
            vf.add_lines_with_ghosts('notbxbfse', [parent_id_utf8], [])
 
725
            vf.add_lines_with_ghosts(b'notbxbfse', [parent_id_utf8], [])
806
726
        except NotImplementedError:
807
727
            # check the other ghost apis are also not implemented
808
 
            self.assertRaises(NotImplementedError, vf.get_ancestry_with_ghosts, ['foo'])
809
 
            self.assertRaises(NotImplementedError, vf.get_parents_with_ghosts, 'foo')
 
728
            self.assertRaises(NotImplementedError,
 
729
                              vf.get_ancestry_with_ghosts, [b'foo'])
 
730
            self.assertRaises(NotImplementedError,
 
731
                              vf.get_parents_with_ghosts, b'foo')
810
732
            return
811
733
        vf = self.reopen_file()
812
734
        # test key graph related apis: getncestry, _graph, get_parents
813
735
        # has_version
814
736
        # - these are ghost unaware and must not be reflect ghosts
815
 
        self.assertEqual(['notbxbfse'], vf.get_ancestry('notbxbfse'))
 
737
        self.assertEqual([b'notbxbfse'], vf.get_ancestry(b'notbxbfse'))
816
738
        self.assertFalse(vf.has_version(parent_id_utf8))
817
739
        # we have _with_ghost apis to give us ghost information.
818
 
        self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry_with_ghosts(['notbxbfse']))
819
 
        self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
 
740
        self.assertEqual([parent_id_utf8, b'notbxbfse'],
 
741
                         vf.get_ancestry_with_ghosts([b'notbxbfse']))
 
742
        self.assertEqual([parent_id_utf8],
 
743
                         vf.get_parents_with_ghosts(b'notbxbfse'))
820
744
        # if we add something that is a ghost of another, it should correct the
821
745
        # results of the prior apis
822
746
        vf.add_lines(parent_id_utf8, [], [])
823
 
        self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry(['notbxbfse']))
824
 
        self.assertEqual({'notbxbfse':(parent_id_utf8,)},
825
 
            vf.get_parent_map(['notbxbfse']))
 
747
        self.assertEqual([parent_id_utf8, b'notbxbfse'],
 
748
                         vf.get_ancestry([b'notbxbfse']))
 
749
        self.assertEqual({b'notbxbfse': (parent_id_utf8,)},
 
750
                         vf.get_parent_map([b'notbxbfse']))
826
751
        self.assertTrue(vf.has_version(parent_id_utf8))
827
752
        # we have _with_ghost apis to give us ghost information.
828
 
        self.assertEqual([parent_id_utf8, 'notbxbfse'],
829
 
            vf.get_ancestry_with_ghosts(['notbxbfse']))
830
 
        self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
 
753
        self.assertEqual([parent_id_utf8, b'notbxbfse'],
 
754
                         vf.get_ancestry_with_ghosts([b'notbxbfse']))
 
755
        self.assertEqual([parent_id_utf8],
 
756
                         vf.get_parents_with_ghosts(b'notbxbfse'))
831
757
 
832
758
    def test_add_lines_with_ghosts_after_normal_revs(self):
833
759
        # some versioned file formats allow lines to be added with parent
837
763
        vf = self.get_file()
838
764
        # probe for ghost support
839
765
        try:
840
 
            vf.add_lines_with_ghosts('base', [], ['line\n', 'line_b\n'])
 
766
            vf.add_lines_with_ghosts(b'base', [], [b'line\n', b'line_b\n'])
841
767
        except NotImplementedError:
842
768
            return
843
 
        vf.add_lines_with_ghosts('references_ghost',
844
 
                                 ['base', 'a_ghost'],
845
 
                                 ['line\n', 'line_b\n', 'line_c\n'])
846
 
        origins = vf.annotate('references_ghost')
847
 
        self.assertEquals(('base', 'line\n'), origins[0])
848
 
        self.assertEquals(('base', 'line_b\n'), origins[1])
849
 
        self.assertEquals(('references_ghost', 'line_c\n'), origins[2])
 
769
        vf.add_lines_with_ghosts(b'references_ghost',
 
770
                                 [b'base', b'a_ghost'],
 
771
                                 [b'line\n', b'line_b\n', b'line_c\n'])
 
772
        origins = vf.annotate(b'references_ghost')
 
773
        self.assertEqual((b'base', b'line\n'), origins[0])
 
774
        self.assertEqual((b'base', b'line_b\n'), origins[1])
 
775
        self.assertEqual((b'references_ghost', b'line_c\n'), origins[2])
850
776
 
851
777
    def test_readonly_mode(self):
852
 
        transport = get_transport(self.get_url('.'))
 
778
        t = self.get_transport()
853
779
        factory = self.get_factory()
854
 
        vf = factory('id', transport, 0777, create=True, access_mode='w')
855
 
        vf = factory('id', transport, access_mode='r')
856
 
        self.assertRaises(errors.ReadOnlyError, vf.add_lines, 'base', [], [])
 
780
        vf = factory('id', t, 0o777, create=True, access_mode='w')
 
781
        vf = factory('id', t, access_mode='r')
 
782
        self.assertRaises(errors.ReadOnlyError, vf.add_lines, b'base', [], [])
857
783
        self.assertRaises(errors.ReadOnlyError,
858
784
                          vf.add_lines_with_ghosts,
859
 
                          'base',
 
785
                          b'base',
860
786
                          [],
861
787
                          [])
862
788
 
864
790
        # check the sha1 data is available
865
791
        vf = self.get_file()
866
792
        # a simple file
867
 
        vf.add_lines('a', [], ['a\n'])
 
793
        vf.add_lines(b'a', [], [b'a\n'])
868
794
        # the same file, different metadata
869
 
        vf.add_lines('b', ['a'], ['a\n'])
 
795
        vf.add_lines(b'b', [b'a'], [b'a\n'])
870
796
        # a file differing only in last newline.
871
 
        vf.add_lines('c', [], ['a'])
 
797
        vf.add_lines(b'c', [], [b'a'])
872
798
        self.assertEqual({
873
 
            'a': '3f786850e387550fdab836ed7e6dc881de23001b',
874
 
            'c': '86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
875
 
            'b': '3f786850e387550fdab836ed7e6dc881de23001b',
 
799
            b'a': b'3f786850e387550fdab836ed7e6dc881de23001b',
 
800
            b'c': b'86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
 
801
            b'b': b'3f786850e387550fdab836ed7e6dc881de23001b',
876
802
            },
877
 
            vf.get_sha1s(['a', 'c', 'b']))
 
803
            vf.get_sha1s([b'a', b'c', b'b']))
878
804
 
879
805
 
880
806
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
881
807
 
882
808
    def get_file(self, name='foo'):
883
 
        return WeaveFile(name, get_transport(self.get_url('.')), create=True,
884
 
            get_scope=self.get_transaction)
 
809
        return WeaveFile(name, self.get_transport(),
 
810
                         create=True,
 
811
                         get_scope=self.get_transaction)
885
812
 
886
813
    def get_file_corrupted_text(self):
887
 
        w = WeaveFile('foo', get_transport(self.get_url('.')), create=True,
888
 
            get_scope=self.get_transaction)
889
 
        w.add_lines('v1', [], ['hello\n'])
890
 
        w.add_lines('v2', ['v1'], ['hello\n', 'there\n'])
 
814
        w = WeaveFile('foo', self.get_transport(),
 
815
                      create=True,
 
816
                      get_scope=self.get_transaction)
 
817
        w.add_lines(b'v1', [], [b'hello\n'])
 
818
        w.add_lines(b'v2', [b'v1'], [b'hello\n', b'there\n'])
891
819
 
892
820
        # We are going to invasively corrupt the text
893
821
        # Make sure the internals of weave are the same
894
 
        self.assertEqual([('{', 0)
895
 
                        , 'hello\n'
896
 
                        , ('}', None)
897
 
                        , ('{', 1)
898
 
                        , 'there\n'
899
 
                        , ('}', None)
900
 
                        ], w._weave)
 
822
        self.assertEqual([(b'{', 0), b'hello\n', (b'}', None), (b'{', 1), b'there\n', (b'}', None)
 
823
                          ], w._weave)
901
824
 
902
 
        self.assertEqual(['f572d396fae9206628714fb2ce00f72e94f2258f'
903
 
                        , '90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
904
 
                        ], w._sha1s)
 
825
        self.assertEqual([b'f572d396fae9206628714fb2ce00f72e94f2258f', b'90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
 
826
                          ], w._sha1s)
905
827
        w.check()
906
828
 
907
829
        # Corrupted
908
 
        w._weave[4] = 'There\n'
 
830
        w._weave[4] = b'There\n'
909
831
        return w
910
832
 
911
833
    def get_file_corrupted_checksum(self):
912
834
        w = self.get_file_corrupted_text()
913
835
        # Corrected
914
 
        w._weave[4] = 'there\n'
915
 
        self.assertEqual('hello\nthere\n', w.get_text('v2'))
 
836
        w._weave[4] = b'there\n'
 
837
        self.assertEqual(b'hello\nthere\n', w.get_text(b'v2'))
916
838
 
917
 
        #Invalid checksum, first digit changed
918
 
        w._sha1s[1] =  'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
 
839
        # Invalid checksum, first digit changed
 
840
        w._sha1s[1] = b'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
919
841
        return w
920
842
 
921
843
    def reopen_file(self, name='foo', create=False):
922
 
        return WeaveFile(name, get_transport(self.get_url('.')), create=create,
923
 
            get_scope=self.get_transaction)
 
844
        return WeaveFile(name, self.get_transport(),
 
845
                         create=create,
 
846
                         get_scope=self.get_transaction)
924
847
 
925
848
    def test_no_implicit_create(self):
926
849
        self.assertRaises(errors.NoSuchFile,
927
850
                          WeaveFile,
928
851
                          'foo',
929
 
                          get_transport(self.get_url('.')),
 
852
                          self.get_transport(),
930
853
                          get_scope=self.get_transaction)
931
854
 
932
855
    def get_factory(self):
936
859
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
937
860
 
938
861
    def setUp(self):
939
 
        TestCaseWithMemoryTransport.setUp(self)
 
862
        super(TestPlanMergeVersionedFile, self).setUp()
940
863
        mapper = PrefixMapper()
941
864
        factory = make_file_factory(True, mapper)
942
865
        self.vf1 = factory(self.get_transport('root-1'))
945
868
        self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
946
869
 
947
870
    def test_add_lines(self):
948
 
        self.plan_merge_vf.add_lines(('root', 'a:'), [], [])
949
 
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
950
 
            ('root', 'a'), [], [])
951
 
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
952
 
            ('root', 'a:'), None, [])
953
 
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
954
 
            ('root', 'a:'), [], None)
 
871
        self.plan_merge_vf.add_lines((b'root', b'a:'), [], [])
 
872
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
 
873
                          (b'root', b'a'), [], [])
 
874
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
 
875
                          (b'root', b'a:'), None, [])
 
876
        self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
 
877
                          (b'root', b'a:'), [], None)
955
878
 
956
879
    def setup_abcde(self):
957
 
        self.vf1.add_lines(('root', 'A'), [], ['a'])
958
 
        self.vf1.add_lines(('root', 'B'), [('root', 'A')], ['b'])
959
 
        self.vf2.add_lines(('root', 'C'), [], ['c'])
960
 
        self.vf2.add_lines(('root', 'D'), [('root', 'C')], ['d'])
961
 
        self.plan_merge_vf.add_lines(('root', 'E:'),
962
 
            [('root', 'B'), ('root', 'D')], ['e'])
 
880
        self.vf1.add_lines((b'root', b'A'), [], [b'a'])
 
881
        self.vf1.add_lines((b'root', b'B'), [(b'root', b'A')], [b'b'])
 
882
        self.vf2.add_lines((b'root', b'C'), [], [b'c'])
 
883
        self.vf2.add_lines((b'root', b'D'), [(b'root', b'C')], [b'd'])
 
884
        self.plan_merge_vf.add_lines((b'root', b'E:'),
 
885
                                     [(b'root', b'B'), (b'root', b'D')], [b'e'])
963
886
 
964
887
    def test_get_parents(self):
965
888
        self.setup_abcde()
966
 
        self.assertEqual({('root', 'B'):(('root', 'A'),)},
967
 
            self.plan_merge_vf.get_parent_map([('root', 'B')]))
968
 
        self.assertEqual({('root', 'D'):(('root', 'C'),)},
969
 
            self.plan_merge_vf.get_parent_map([('root', 'D')]))
970
 
        self.assertEqual({('root', 'E:'):(('root', 'B'),('root', 'D'))},
971
 
            self.plan_merge_vf.get_parent_map([('root', 'E:')]))
 
889
        self.assertEqual({(b'root', b'B'): ((b'root', b'A'),)},
 
890
                         self.plan_merge_vf.get_parent_map([(b'root', b'B')]))
 
891
        self.assertEqual({(b'root', b'D'): ((b'root', b'C'),)},
 
892
                         self.plan_merge_vf.get_parent_map([(b'root', b'D')]))
 
893
        self.assertEqual({(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D'))},
 
894
                         self.plan_merge_vf.get_parent_map([(b'root', b'E:')]))
972
895
        self.assertEqual({},
973
 
            self.plan_merge_vf.get_parent_map([('root', 'F')]))
 
896
                         self.plan_merge_vf.get_parent_map([(b'root', b'F')]))
974
897
        self.assertEqual({
975
 
                ('root', 'B'):(('root', 'A'),),
976
 
                ('root', 'D'):(('root', 'C'),),
977
 
                ('root', 'E:'):(('root', 'B'),('root', 'D')),
978
 
                },
 
898
            (b'root', b'B'): ((b'root', b'A'),),
 
899
            (b'root', b'D'): ((b'root', b'C'),),
 
900
            (b'root', b'E:'): ((b'root', b'B'), (b'root', b'D')),
 
901
            },
979
902
            self.plan_merge_vf.get_parent_map(
980
 
                [('root', 'B'), ('root', 'D'), ('root', 'E:'), ('root', 'F')]))
 
903
                [(b'root', b'B'), (b'root', b'D'), (b'root', b'E:'), (b'root', b'F')]))
981
904
 
982
905
    def test_get_record_stream(self):
983
906
        self.setup_abcde()
 
907
 
984
908
        def get_record(suffix):
985
 
            return self.plan_merge_vf.get_record_stream(
986
 
                [('root', suffix)], 'unordered', True).next()
987
 
        self.assertEqual('a', get_record('A').get_bytes_as('fulltext'))
988
 
        self.assertEqual('c', get_record('C').get_bytes_as('fulltext'))
989
 
        self.assertEqual('e', get_record('E:').get_bytes_as('fulltext'))
 
909
            return next(self.plan_merge_vf.get_record_stream(
 
910
                [(b'root', suffix)], 'unordered', True))
 
911
        self.assertEqual(b'a', get_record(b'A').get_bytes_as('fulltext'))
 
912
        self.assertEqual(b'c', get_record(b'C').get_bytes_as('fulltext'))
 
913
        self.assertEqual(b'e', get_record(b'E:').get_bytes_as('fulltext'))
990
914
        self.assertEqual('absent', get_record('F').storage_kind)
991
915
 
992
916
 
999
923
        # we should be able to read from http with a versioned file.
1000
924
        vf = self.get_file()
1001
925
        # try an empty file access
1002
 
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
 
926
        readonly_vf = self.get_factory()('foo',
 
927
                                         transport.get_transport_from_url(self.get_readonly_url('.')))
1003
928
        self.assertEqual([], readonly_vf.versions())
 
929
 
 
930
    def test_readonly_http_works_with_feeling(self):
 
931
        # we should be able to read from http with a versioned file.
 
932
        vf = self.get_file()
1004
933
        # now with feeling.
1005
 
        vf.add_lines('1', [], ['a\n'])
1006
 
        vf.add_lines('2', ['1'], ['b\n', 'a\n'])
1007
 
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
1008
 
        self.assertEqual(['1', '2'], vf.versions())
 
934
        vf.add_lines(b'1', [], [b'a\n'])
 
935
        vf.add_lines(b'2', [b'1'], [b'b\n', b'a\n'])
 
936
        readonly_vf = self.get_factory()('foo',
 
937
                                         transport.get_transport_from_url(self.get_readonly_url('.')))
 
938
        self.assertEqual([b'1', b'2'], vf.versions())
 
939
        self.assertEqual([b'1', b'2'], readonly_vf.versions())
1009
940
        for version in readonly_vf.versions():
1010
941
            readonly_vf.get_lines(version)
1011
942
 
1013
944
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
1014
945
 
1015
946
    def get_file(self):
1016
 
        return WeaveFile('foo', get_transport(self.get_url('.')), create=True,
1017
 
            get_scope=self.get_transaction)
 
947
        return WeaveFile('foo', self.get_transport(),
 
948
                         create=True,
 
949
                         get_scope=self.get_transaction)
1018
950
 
1019
951
    def get_factory(self):
1020
952
        return WeaveFile
1023
955
class MergeCasesMixin(object):
1024
956
 
1025
957
    def doMerge(self, base, a, b, mp):
1026
 
        from cStringIO import StringIO
1027
958
        from textwrap import dedent
1028
959
 
1029
960
        def addcrlf(x):
1030
 
            return x + '\n'
 
961
            return x + b'\n'
1031
962
 
1032
963
        w = self.get_file()
1033
 
        w.add_lines('text0', [], map(addcrlf, base))
1034
 
        w.add_lines('text1', ['text0'], map(addcrlf, a))
1035
 
        w.add_lines('text2', ['text0'], map(addcrlf, b))
 
964
        w.add_lines(b'text0', [], list(map(addcrlf, base)))
 
965
        w.add_lines(b'text1', [b'text0'], list(map(addcrlf, a)))
 
966
        w.add_lines(b'text2', [b'text0'], list(map(addcrlf, b)))
1036
967
 
1037
968
        self.log_contents(w)
1038
969
 
1039
970
        self.log('merge plan:')
1040
 
        p = list(w.plan_merge('text1', 'text2'))
 
971
        p = list(w.plan_merge(b'text1', b'text2'))
1041
972
        for state, line in p:
1042
973
            if line:
1043
974
                self.log('%12s | %s' % (state, line[:-1]))
1044
975
 
1045
976
        self.log('merge:')
1046
 
        mt = StringIO()
 
977
        mt = BytesIO()
1047
978
        mt.writelines(w.weave_merge(p))
1048
979
        mt.seek(0)
1049
980
        self.log(mt.getvalue())
1050
981
 
1051
 
        mp = map(addcrlf, mp)
 
982
        mp = list(map(addcrlf, mp))
1052
983
        self.assertEqual(mt.readlines(), mp)
1053
984
 
1054
 
 
1055
985
    def testOneInsert(self):
1056
986
        self.doMerge([],
1057
 
                     ['aa'],
 
987
                     [b'aa'],
1058
988
                     [],
1059
 
                     ['aa'])
 
989
                     [b'aa'])
1060
990
 
1061
991
    def testSeparateInserts(self):
1062
 
        self.doMerge(['aaa', 'bbb', 'ccc'],
1063
 
                     ['aaa', 'xxx', 'bbb', 'ccc'],
1064
 
                     ['aaa', 'bbb', 'yyy', 'ccc'],
1065
 
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
 
992
        self.doMerge([b'aaa', b'bbb', b'ccc'],
 
993
                     [b'aaa', b'xxx', b'bbb', b'ccc'],
 
994
                     [b'aaa', b'bbb', b'yyy', b'ccc'],
 
995
                     [b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
1066
996
 
1067
997
    def testSameInsert(self):
1068
 
        self.doMerge(['aaa', 'bbb', 'ccc'],
1069
 
                     ['aaa', 'xxx', 'bbb', 'ccc'],
1070
 
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'],
1071
 
                     ['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
1072
 
    overlappedInsertExpected = ['aaa', 'xxx', 'yyy', 'bbb']
 
998
        self.doMerge([b'aaa', b'bbb', b'ccc'],
 
999
                     [b'aaa', b'xxx', b'bbb', b'ccc'],
 
1000
                     [b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'],
 
1001
                     [b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
 
1002
    overlappedInsertExpected = [b'aaa', b'xxx', b'yyy', b'bbb']
 
1003
 
1073
1004
    def testOverlappedInsert(self):
1074
 
        self.doMerge(['aaa', 'bbb'],
1075
 
                     ['aaa', 'xxx', 'yyy', 'bbb'],
1076
 
                     ['aaa', 'xxx', 'bbb'], self.overlappedInsertExpected)
 
1005
        self.doMerge([b'aaa', b'bbb'],
 
1006
                     [b'aaa', b'xxx', b'yyy', b'bbb'],
 
1007
                     [b'aaa', b'xxx', b'bbb'], self.overlappedInsertExpected)
1077
1008
 
1078
1009
        # really it ought to reduce this to
1079
 
        # ['aaa', 'xxx', 'yyy', 'bbb']
1080
 
 
 
1010
        # [b'aaa', b'xxx', b'yyy', b'bbb']
1081
1011
 
1082
1012
    def testClashReplace(self):
1083
 
        self.doMerge(['aaa'],
1084
 
                     ['xxx'],
1085
 
                     ['yyy', 'zzz'],
1086
 
                     ['<<<<<<< ', 'xxx', '=======', 'yyy', 'zzz',
1087
 
                      '>>>>>>> '])
 
1013
        self.doMerge([b'aaa'],
 
1014
                     [b'xxx'],
 
1015
                     [b'yyy', b'zzz'],
 
1016
                     [b'<<<<<<< ', b'xxx', b'=======', b'yyy', b'zzz',
 
1017
                      b'>>>>>>> '])
1088
1018
 
1089
1019
    def testNonClashInsert1(self):
1090
 
        self.doMerge(['aaa'],
1091
 
                     ['xxx', 'aaa'],
1092
 
                     ['yyy', 'zzz'],
1093
 
                     ['<<<<<<< ', 'xxx', 'aaa', '=======', 'yyy', 'zzz',
1094
 
                      '>>>>>>> '])
 
1020
        self.doMerge([b'aaa'],
 
1021
                     [b'xxx', b'aaa'],
 
1022
                     [b'yyy', b'zzz'],
 
1023
                     [b'<<<<<<< ', b'xxx', b'aaa', b'=======', b'yyy', b'zzz',
 
1024
                      b'>>>>>>> '])
1095
1025
 
1096
1026
    def testNonClashInsert2(self):
1097
 
        self.doMerge(['aaa'],
1098
 
                     ['aaa'],
1099
 
                     ['yyy', 'zzz'],
1100
 
                     ['yyy', 'zzz'])
1101
 
 
 
1027
        self.doMerge([b'aaa'],
 
1028
                     [b'aaa'],
 
1029
                     [b'yyy', b'zzz'],
 
1030
                     [b'yyy', b'zzz'])
1102
1031
 
1103
1032
    def testDeleteAndModify(self):
1104
1033
        """Clashing delete and modification.
1111
1040
        # skippd, not working yet
1112
1041
        return
1113
1042
 
1114
 
        self.doMerge(['aaa', 'bbb', 'ccc'],
1115
 
                     ['aaa', 'ddd', 'ccc'],
1116
 
                     ['aaa', 'ccc'],
1117
 
                     ['<<<<<<<< ', 'aaa', '=======', '>>>>>>> ', 'ccc'])
 
1043
        self.doMerge([b'aaa', b'bbb', b'ccc'],
 
1044
                     [b'aaa', b'ddd', b'ccc'],
 
1045
                     [b'aaa', b'ccc'],
 
1046
                     [b'<<<<<<<< ', b'aaa', b'=======', b'>>>>>>> ', b'ccc'])
1118
1047
 
1119
1048
    def _test_merge_from_strings(self, base, a, b, expected):
1120
1049
        w = self.get_file()
1121
 
        w.add_lines('text0', [], base.splitlines(True))
1122
 
        w.add_lines('text1', ['text0'], a.splitlines(True))
1123
 
        w.add_lines('text2', ['text0'], b.splitlines(True))
 
1050
        w.add_lines(b'text0', [], base.splitlines(True))
 
1051
        w.add_lines(b'text1', [b'text0'], a.splitlines(True))
 
1052
        w.add_lines(b'text2', [b'text0'], b.splitlines(True))
1124
1053
        self.log('merge plan:')
1125
 
        p = list(w.plan_merge('text1', 'text2'))
 
1054
        p = list(w.plan_merge(b'text1', b'text2'))
1126
1055
        for state, line in p:
1127
1056
            if line:
1128
1057
                self.log('%12s | %s' % (state, line[:-1]))
1129
1058
        self.log('merge result:')
1130
 
        result_text = ''.join(w.weave_merge(p))
 
1059
        result_text = b''.join(w.weave_merge(p))
1131
1060
        self.log(result_text)
1132
1061
        self.assertEqualDiff(result_text, expected)
1133
1062
 
1134
1063
    def test_weave_merge_conflicts(self):
1135
1064
        # does weave merge properly handle plans that end with unchanged?
1136
 
        result = ''.join(self.get_file().weave_merge([('new-a', 'hello\n')]))
1137
 
        self.assertEqual(result, 'hello\n')
 
1065
        result = b''.join(self.get_file().weave_merge([('new-a', b'hello\n')]))
 
1066
        self.assertEqual(result, b'hello\n')
1138
1067
 
1139
1068
    def test_deletion_extended(self):
1140
1069
        """One side deletes, the other deletes more.
1141
1070
        """
1142
 
        base = """\
 
1071
        base = b"""\
1143
1072
            line 1
1144
1073
            line 2
1145
1074
            line 3
1146
1075
            """
1147
 
        a = """\
 
1076
        a = b"""\
1148
1077
            line 1
1149
1078
            line 2
1150
1079
            """
1151
 
        b = """\
 
1080
        b = b"""\
1152
1081
            line 1
1153
1082
            """
1154
 
        result = """\
 
1083
        result = b"""\
1155
1084
            line 1
1156
1085
<<<<<<<\x20
1157
1086
            line 2
1166
1095
        Arguably it'd be better to treat these as agreement, rather than
1167
1096
        conflict, but for now conflict is safer.
1168
1097
        """
1169
 
        base = """\
 
1098
        base = b"""\
1170
1099
            start context
1171
1100
            int a() {}
1172
1101
            int b() {}
1173
1102
            int c() {}
1174
1103
            end context
1175
1104
            """
1176
 
        a = """\
 
1105
        a = b"""\
1177
1106
            start context
1178
1107
            int a() {}
1179
1108
            end context
1180
1109
            """
1181
 
        b = """\
 
1110
        b = b"""\
1182
1111
            start context
1183
1112
            int c() {}
1184
1113
            end context
1185
1114
            """
1186
 
        result = """\
 
1115
        result = b"""\
1187
1116
            start context
1188
1117
<<<<<<<\x20
1189
1118
            int a() {}
1196
1125
 
1197
1126
    def test_agreement_deletion(self):
1198
1127
        """Agree to delete some lines, without conflicts."""
1199
 
        base = """\
 
1128
        base = b"""\
1200
1129
            start context
1201
1130
            base line 1
1202
1131
            base line 2
1203
1132
            end context
1204
1133
            """
1205
 
        a = """\
1206
 
            start context
1207
 
            base line 1
1208
 
            end context
1209
 
            """
1210
 
        b = """\
1211
 
            start context
1212
 
            base line 1
1213
 
            end context
1214
 
            """
1215
 
        result = """\
 
1134
        a = b"""\
 
1135
            start context
 
1136
            base line 1
 
1137
            end context
 
1138
            """
 
1139
        b = b"""\
 
1140
            start context
 
1141
            base line 1
 
1142
            end context
 
1143
            """
 
1144
        result = b"""\
1216
1145
            start context
1217
1146
            base line 1
1218
1147
            end context
1229
1158
 
1230
1159
        It's better to consider the whole thing as a disagreement region.
1231
1160
        """
1232
 
        base = """\
 
1161
        base = b"""\
1233
1162
            start context
1234
1163
            base line 1
1235
1164
            base line 2
1236
1165
            end context
1237
1166
            """
1238
 
        a = """\
 
1167
        a = b"""\
1239
1168
            start context
1240
1169
            base line 1
1241
1170
            a's replacement line 2
1242
1171
            end context
1243
1172
            """
1244
 
        b = """\
 
1173
        b = b"""\
1245
1174
            start context
1246
1175
            b replaces
1247
1176
            both lines
1248
1177
            end context
1249
1178
            """
1250
 
        result = """\
 
1179
        result = b"""\
1251
1180
            start context
1252
1181
<<<<<<<\x20
1253
1182
            base line 1
1264
1193
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1265
1194
 
1266
1195
    def get_file(self, name='foo'):
1267
 
        return WeaveFile(name, get_transport(self.get_url('.')), create=True)
 
1196
        return WeaveFile(name, self.get_transport(),
 
1197
                         create=True)
1268
1198
 
1269
1199
    def log_contents(self, w):
1270
1200
        self.log('weave is:')
1271
 
        tmpf = StringIO()
 
1201
        tmpf = BytesIO()
1272
1202
        write_weave(w, tmpf)
1273
1203
        self.log(tmpf.getvalue())
1274
1204
 
1275
 
    overlappedInsertExpected = ['aaa', '<<<<<<< ', 'xxx', 'yyy', '=======',
1276
 
                                'xxx', '>>>>>>> ', 'bbb']
 
1205
    overlappedInsertExpected = [b'aaa', b'<<<<<<< ', b'xxx', b'yyy', b'=======',
 
1206
                                b'xxx', b'>>>>>>> ', b'bbb']
1277
1207
 
1278
1208
 
1279
1209
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1308
1238
    def helpGetBytes(self, f, ft_adapter, delta_adapter):
1309
1239
        """Grab the interested adapted texts for tests."""
1310
1240
        # origin is a fulltext
1311
 
        entries = f.get_record_stream([('origin',)], 'unordered', False)
1312
 
        base = entries.next()
 
1241
        entries = f.get_record_stream([(b'origin',)], 'unordered', False)
 
1242
        base = next(entries)
1313
1243
        ft_data = ft_adapter.get_bytes(base)
1314
1244
        # merged is both a delta and multiple parents.
1315
 
        entries = f.get_record_stream([('merged',)], 'unordered', False)
1316
 
        merged = entries.next()
 
1245
        entries = f.get_record_stream([(b'merged',)], 'unordered', False)
 
1246
        merged = next(entries)
1317
1247
        delta_data = delta_adapter.get_bytes(merged)
1318
1248
        return ft_data, delta_data
1319
1249
 
1323
1253
        f = self.get_knit()
1324
1254
        get_diamond_files(f, 1, trailing_eol=False)
1325
1255
        ft_data, delta_data = self.helpGetBytes(f,
1326
 
            _mod_knit.FTAnnotatedToUnannotated(None),
1327
 
            _mod_knit.DeltaAnnotatedToUnannotated(None))
1328
 
        self.assertEqual(
1329
 
            'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1330
 
            'origin\n'
1331
 
            'end origin\n',
1332
 
            GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1333
 
        self.assertEqual(
1334
 
            'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1335
 
            '1,2,3\nleft\nright\nmerged\nend merged\n',
1336
 
            GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
 
1256
                                                _mod_knit.FTAnnotatedToUnannotated(
 
1257
                                                    None),
 
1258
                                                _mod_knit.DeltaAnnotatedToUnannotated(None))
 
1259
        self.assertEqual(
 
1260
            b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
 
1261
            b'origin\n'
 
1262
            b'end origin\n',
 
1263
            GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
 
1264
        self.assertEqual(
 
1265
            b'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
 
1266
            b'1,2,3\nleft\nright\nmerged\nend merged\n',
 
1267
            GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1337
1268
 
1338
1269
    def test_deannotation(self):
1339
1270
        """Test converting annotated knits to unannotated knits."""
1341
1272
        f = self.get_knit()
1342
1273
        get_diamond_files(f, 1)
1343
1274
        ft_data, delta_data = self.helpGetBytes(f,
1344
 
            _mod_knit.FTAnnotatedToUnannotated(None),
1345
 
            _mod_knit.DeltaAnnotatedToUnannotated(None))
1346
 
        self.assertEqual(
1347
 
            'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1348
 
            'origin\n'
1349
 
            'end origin\n',
1350
 
            GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1351
 
        self.assertEqual(
1352
 
            'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1353
 
            '2,2,2\nright\nmerged\nend merged\n',
1354
 
            GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
 
1275
                                                _mod_knit.FTAnnotatedToUnannotated(
 
1276
                                                    None),
 
1277
                                                _mod_knit.DeltaAnnotatedToUnannotated(None))
 
1278
        self.assertEqual(
 
1279
            b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
 
1280
            b'origin\n'
 
1281
            b'end origin\n',
 
1282
            GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
 
1283
        self.assertEqual(
 
1284
            b'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
 
1285
            b'2,2,2\nright\nmerged\nend merged\n',
 
1286
            GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1355
1287
 
1356
1288
    def test_annotated_to_fulltext_no_eol(self):
1357
1289
        """Test adapting annotated knits to full texts (for -> weaves)."""
1362
1294
        # must have the base lines requested from it.
1363
1295
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1364
1296
        ft_data, delta_data = self.helpGetBytes(f,
1365
 
            _mod_knit.FTAnnotatedToFullText(None),
1366
 
            _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1367
 
        self.assertEqual('origin', ft_data)
1368
 
        self.assertEqual('base\nleft\nright\nmerged', delta_data)
1369
 
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1370
 
            True)], logged_vf.calls)
 
1297
                                                _mod_knit.FTAnnotatedToFullText(
 
1298
                                                    None),
 
1299
                                                _mod_knit.DeltaAnnotatedToFullText(logged_vf))
 
1300
        self.assertEqual(b'origin', ft_data)
 
1301
        self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
 
1302
        self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
 
1303
                           True)], logged_vf.calls)
1371
1304
 
1372
1305
    def test_annotated_to_fulltext(self):
1373
1306
        """Test adapting annotated knits to full texts (for -> weaves)."""
1378
1311
        # must have the base lines requested from it.
1379
1312
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1380
1313
        ft_data, delta_data = self.helpGetBytes(f,
1381
 
            _mod_knit.FTAnnotatedToFullText(None),
1382
 
            _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1383
 
        self.assertEqual('origin\n', ft_data)
1384
 
        self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1385
 
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1386
 
            True)], logged_vf.calls)
 
1314
                                                _mod_knit.FTAnnotatedToFullText(
 
1315
                                                    None),
 
1316
                                                _mod_knit.DeltaAnnotatedToFullText(logged_vf))
 
1317
        self.assertEqual(b'origin\n', ft_data)
 
1318
        self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
 
1319
        self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
 
1320
                           True)], logged_vf.calls)
1387
1321
 
1388
1322
    def test_unannotated_to_fulltext(self):
1389
1323
        """Test adapting unannotated knits to full texts.
1397
1331
        # must have the base lines requested from it.
1398
1332
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1399
1333
        ft_data, delta_data = self.helpGetBytes(f,
1400
 
            _mod_knit.FTPlainToFullText(None),
1401
 
            _mod_knit.DeltaPlainToFullText(logged_vf))
1402
 
        self.assertEqual('origin\n', ft_data)
1403
 
        self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1404
 
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1405
 
            True)], logged_vf.calls)
 
1334
                                                _mod_knit.FTPlainToFullText(
 
1335
                                                    None),
 
1336
                                                _mod_knit.DeltaPlainToFullText(logged_vf))
 
1337
        self.assertEqual(b'origin\n', ft_data)
 
1338
        self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
 
1339
        self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
 
1340
                           True)], logged_vf.calls)
1406
1341
 
1407
1342
    def test_unannotated_to_fulltext_no_eol(self):
1408
1343
        """Test adapting unannotated knits to full texts.
1416
1351
        # must have the base lines requested from it.
1417
1352
        logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1418
1353
        ft_data, delta_data = self.helpGetBytes(f,
1419
 
            _mod_knit.FTPlainToFullText(None),
1420
 
            _mod_knit.DeltaPlainToFullText(logged_vf))
1421
 
        self.assertEqual('origin', ft_data)
1422
 
        self.assertEqual('base\nleft\nright\nmerged', delta_data)
1423
 
        self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1424
 
            True)], logged_vf.calls)
 
1354
                                                _mod_knit.FTPlainToFullText(
 
1355
                                                    None),
 
1356
                                                _mod_knit.DeltaPlainToFullText(logged_vf))
 
1357
        self.assertEqual(b'origin', ft_data)
 
1358
        self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
 
1359
        self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
 
1360
                           True)], logged_vf.calls)
1425
1361
 
1426
1362
 
1427
1363
class TestKeyMapper(TestCaseWithMemoryTransport):
1429
1365
 
1430
1366
    def test_identity_mapper(self):
1431
1367
        mapper = versionedfile.ConstantMapper("inventory")
1432
 
        self.assertEqual("inventory", mapper.map(('foo@ar',)))
1433
 
        self.assertEqual("inventory", mapper.map(('quux',)))
 
1368
        self.assertEqual("inventory", mapper.map((b'foo@ar',)))
 
1369
        self.assertEqual("inventory", mapper.map((b'quux',)))
1434
1370
 
1435
1371
    def test_prefix_mapper(self):
1436
1372
        #format5: plain
1437
1373
        mapper = versionedfile.PrefixMapper()
1438
 
        self.assertEqual("file-id", mapper.map(("file-id", "revision-id")))
1439
 
        self.assertEqual("new-id", mapper.map(("new-id", "revision-id")))
1440
 
        self.assertEqual(('file-id',), mapper.unmap("file-id"))
1441
 
        self.assertEqual(('new-id',), mapper.unmap("new-id"))
 
1374
        self.assertEqual("file-id", mapper.map((b"file-id", b"revision-id")))
 
1375
        self.assertEqual("new-id", mapper.map((b"new-id", b"revision-id")))
 
1376
        self.assertEqual((b'file-id',), mapper.unmap("file-id"))
 
1377
        self.assertEqual((b'new-id',), mapper.unmap("new-id"))
1442
1378
 
1443
1379
    def test_hash_prefix_mapper(self):
1444
1380
        #format6: hash + plain
1445
1381
        mapper = versionedfile.HashPrefixMapper()
1446
 
        self.assertEqual("9b/file-id", mapper.map(("file-id", "revision-id")))
1447
 
        self.assertEqual("45/new-id", mapper.map(("new-id", "revision-id")))
1448
 
        self.assertEqual(('file-id',), mapper.unmap("9b/file-id"))
1449
 
        self.assertEqual(('new-id',), mapper.unmap("45/new-id"))
 
1382
        self.assertEqual(
 
1383
            "9b/file-id", mapper.map((b"file-id", b"revision-id")))
 
1384
        self.assertEqual("45/new-id", mapper.map((b"new-id", b"revision-id")))
 
1385
        self.assertEqual((b'file-id',), mapper.unmap("9b/file-id"))
 
1386
        self.assertEqual((b'new-id',), mapper.unmap("45/new-id"))
1450
1387
 
1451
1388
    def test_hash_escaped_mapper(self):
1452
1389
        #knit1: hash + escaped
1453
1390
        mapper = versionedfile.HashEscapedPrefixMapper()
1454
 
        self.assertEqual("88/%2520", mapper.map((" ", "revision-id")))
1455
 
        self.assertEqual("ed/fil%2545-%2549d", mapper.map(("filE-Id",
1456
 
            "revision-id")))
1457
 
        self.assertEqual("88/ne%2557-%2549d", mapper.map(("neW-Id",
1458
 
            "revision-id")))
1459
 
        self.assertEqual(('filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1460
 
        self.assertEqual(('neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
 
1391
        self.assertEqual("88/%2520", mapper.map((b" ", b"revision-id")))
 
1392
        self.assertEqual("ed/fil%2545-%2549d", mapper.map((b"filE-Id",
 
1393
                                                           b"revision-id")))
 
1394
        self.assertEqual("88/ne%2557-%2549d", mapper.map((b"neW-Id",
 
1395
                                                          b"revision-id")))
 
1396
        self.assertEqual((b'filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
 
1397
        self.assertEqual((b'neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1461
1398
 
1462
1399
 
1463
1400
class TestVersionedFiles(TestCaseWithMemoryTransport):
1464
1401
    """Tests for the multiple-file variant of VersionedFile."""
1465
1402
 
 
1403
    # We want to be sure of behaviour for:
 
1404
    # weaves prefix layout (weave texts)
 
1405
    # individually named weaves (weave inventories)
 
1406
    # annotated knits - prefix|hash|hash-escape layout, we test the third only
 
1407
    #                   as it is the most complex mapper.
 
1408
    # individually named knits
 
1409
    # individual no-graph knits in packs (signatures)
 
1410
    # individual graph knits in packs (inventories)
 
1411
    # individual graph nocompression knits in packs (revisions)
 
1412
    # plain text knits in packs (texts)
 
1413
    len_one_scenarios = [
 
1414
        ('weave-named', {
 
1415
            'cleanup': None,
 
1416
            'factory': make_versioned_files_factory(WeaveFile,
 
1417
                                                    ConstantMapper('inventory')),
 
1418
            'graph': True,
 
1419
            'key_length': 1,
 
1420
            'support_partial_insertion': False,
 
1421
            }),
 
1422
        ('named-knit', {
 
1423
            'cleanup': None,
 
1424
            'factory': make_file_factory(False, ConstantMapper('revisions')),
 
1425
            'graph': True,
 
1426
            'key_length': 1,
 
1427
            'support_partial_insertion': False,
 
1428
            }),
 
1429
        ('named-nograph-nodelta-knit-pack', {
 
1430
            'cleanup': cleanup_pack_knit,
 
1431
            'factory': make_pack_factory(False, False, 1),
 
1432
            'graph': False,
 
1433
            'key_length': 1,
 
1434
            'support_partial_insertion': False,
 
1435
            }),
 
1436
        ('named-graph-knit-pack', {
 
1437
            'cleanup': cleanup_pack_knit,
 
1438
            'factory': make_pack_factory(True, True, 1),
 
1439
            'graph': True,
 
1440
            'key_length': 1,
 
1441
            'support_partial_insertion': True,
 
1442
            }),
 
1443
        ('named-graph-nodelta-knit-pack', {
 
1444
            'cleanup': cleanup_pack_knit,
 
1445
            'factory': make_pack_factory(True, False, 1),
 
1446
            'graph': True,
 
1447
            'key_length': 1,
 
1448
            'support_partial_insertion': False,
 
1449
            }),
 
1450
        ('groupcompress-nograph', {
 
1451
            'cleanup': groupcompress.cleanup_pack_group,
 
1452
            'factory': groupcompress.make_pack_factory(False, False, 1),
 
1453
            'graph': False,
 
1454
            'key_length': 1,
 
1455
            'support_partial_insertion': False,
 
1456
            }),
 
1457
        ]
 
1458
    len_two_scenarios = [
 
1459
        ('weave-prefix', {
 
1460
            'cleanup': None,
 
1461
            'factory': make_versioned_files_factory(WeaveFile,
 
1462
                                                    PrefixMapper()),
 
1463
            'graph': True,
 
1464
            'key_length': 2,
 
1465
            'support_partial_insertion': False,
 
1466
            }),
 
1467
        ('annotated-knit-escape', {
 
1468
            'cleanup': None,
 
1469
            'factory': make_file_factory(True, HashEscapedPrefixMapper()),
 
1470
            'graph': True,
 
1471
            'key_length': 2,
 
1472
            'support_partial_insertion': False,
 
1473
            }),
 
1474
        ('plain-knit-pack', {
 
1475
            'cleanup': cleanup_pack_knit,
 
1476
            'factory': make_pack_factory(True, True, 2),
 
1477
            'graph': True,
 
1478
            'key_length': 2,
 
1479
            'support_partial_insertion': True,
 
1480
            }),
 
1481
        ('groupcompress', {
 
1482
            'cleanup': groupcompress.cleanup_pack_group,
 
1483
            'factory': groupcompress.make_pack_factory(True, False, 1),
 
1484
            'graph': True,
 
1485
            'key_length': 1,
 
1486
            'support_partial_insertion': False,
 
1487
            }),
 
1488
        ]
 
1489
 
 
1490
    scenarios = len_one_scenarios + len_two_scenarios
 
1491
 
1466
1492
    def get_versionedfiles(self, relpath='files'):
1467
1493
        transport = self.get_transport(relpath)
1468
1494
        if relpath != '.':
1477
1503
        if self.key_length == 1:
1478
1504
            return (suffix,)
1479
1505
        else:
1480
 
            return ('FileA',) + (suffix,)
 
1506
            return (b'FileA',) + (suffix,)
 
1507
 
 
1508
    def test_add_fallback_implies_without_fallbacks(self):
 
1509
        f = self.get_versionedfiles('files')
 
1510
        if getattr(f, 'add_fallback_versioned_files', None) is None:
 
1511
            raise TestNotApplicable("%s doesn't support fallbacks"
 
1512
                                    % (f.__class__.__name__,))
 
1513
        g = self.get_versionedfiles('fallback')
 
1514
        key_a = self.get_simple_key(b'a')
 
1515
        g.add_lines(key_a, [], [b'\n'])
 
1516
        f.add_fallback_versioned_files(g)
 
1517
        self.assertTrue(key_a in f.get_parent_map([key_a]))
 
1518
        self.assertFalse(
 
1519
            key_a in f.without_fallbacks().get_parent_map([key_a]))
1481
1520
 
1482
1521
    def test_add_lines(self):
1483
1522
        f = self.get_versionedfiles()
1484
 
        key0 = self.get_simple_key('r0')
1485
 
        key1 = self.get_simple_key('r1')
1486
 
        key2 = self.get_simple_key('r2')
1487
 
        keyf = self.get_simple_key('foo')
1488
 
        f.add_lines(key0, [], ['a\n', 'b\n'])
1489
 
        if self.graph:
1490
 
            f.add_lines(key1, [key0], ['b\n', 'c\n'])
1491
 
        else:
1492
 
            f.add_lines(key1, [], ['b\n', 'c\n'])
1493
 
        keys = f.keys()
1494
 
        self.assertTrue(key0 in keys)
1495
 
        self.assertTrue(key1 in keys)
1496
 
        records = []
1497
 
        for record in f.get_record_stream([key0, key1], 'unordered', True):
1498
 
            records.append((record.key, record.get_bytes_as('fulltext')))
1499
 
        records.sort()
1500
 
        self.assertEqual([(key0, 'a\nb\n'), (key1, 'b\nc\n')], records)
1501
 
 
1502
 
    def test__add_text(self):
1503
 
        f = self.get_versionedfiles()
1504
 
        key0 = self.get_simple_key('r0')
1505
 
        key1 = self.get_simple_key('r1')
1506
 
        key2 = self.get_simple_key('r2')
1507
 
        keyf = self.get_simple_key('foo')
1508
 
        f._add_text(key0, [], 'a\nb\n')
1509
 
        if self.graph:
1510
 
            f._add_text(key1, [key0], 'b\nc\n')
1511
 
        else:
1512
 
            f._add_text(key1, [], 'b\nc\n')
1513
 
        keys = f.keys()
1514
 
        self.assertTrue(key0 in keys)
1515
 
        self.assertTrue(key1 in keys)
1516
 
        records = []
1517
 
        for record in f.get_record_stream([key0, key1], 'unordered', True):
1518
 
            records.append((record.key, record.get_bytes_as('fulltext')))
1519
 
        records.sort()
1520
 
        self.assertEqual([(key0, 'a\nb\n'), (key1, 'b\nc\n')], records)
 
1523
        key0 = self.get_simple_key(b'r0')
 
1524
        key1 = self.get_simple_key(b'r1')
 
1525
        key2 = self.get_simple_key(b'r2')
 
1526
        keyf = self.get_simple_key(b'foo')
 
1527
        f.add_lines(key0, [], [b'a\n', b'b\n'])
 
1528
        if self.graph:
 
1529
            f.add_lines(key1, [key0], [b'b\n', b'c\n'])
 
1530
        else:
 
1531
            f.add_lines(key1, [], [b'b\n', b'c\n'])
 
1532
        keys = f.keys()
 
1533
        self.assertTrue(key0 in keys)
 
1534
        self.assertTrue(key1 in keys)
 
1535
        records = []
 
1536
        for record in f.get_record_stream([key0, key1], 'unordered', True):
 
1537
            records.append((record.key, record.get_bytes_as('fulltext')))
 
1538
        records.sort()
 
1539
        self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1521
1540
 
1522
1541
    def test_annotate(self):
1523
1542
        files = self.get_versionedfiles()
1525
1544
        if self.key_length == 1:
1526
1545
            prefix = ()
1527
1546
        else:
1528
 
            prefix = ('FileA',)
 
1547
            prefix = (b'FileA',)
1529
1548
        # introduced full text
1530
 
        origins = files.annotate(prefix + ('origin',))
 
1549
        origins = files.annotate(prefix + (b'origin',))
1531
1550
        self.assertEqual([
1532
 
            (prefix + ('origin',), 'origin\n')],
 
1551
            (prefix + (b'origin',), b'origin\n')],
1533
1552
            origins)
1534
1553
        # a delta
1535
 
        origins = files.annotate(prefix + ('base',))
 
1554
        origins = files.annotate(prefix + (b'base',))
1536
1555
        self.assertEqual([
1537
 
            (prefix + ('base',), 'base\n')],
 
1556
            (prefix + (b'base',), b'base\n')],
1538
1557
            origins)
1539
1558
        # a merge
1540
 
        origins = files.annotate(prefix + ('merged',))
 
1559
        origins = files.annotate(prefix + (b'merged',))
1541
1560
        if self.graph:
1542
1561
            self.assertEqual([
1543
 
                (prefix + ('base',), 'base\n'),
1544
 
                (prefix + ('left',), 'left\n'),
1545
 
                (prefix + ('right',), 'right\n'),
1546
 
                (prefix + ('merged',), 'merged\n')
 
1562
                (prefix + (b'base',), b'base\n'),
 
1563
                (prefix + (b'left',), b'left\n'),
 
1564
                (prefix + (b'right',), b'right\n'),
 
1565
                (prefix + (b'merged',), b'merged\n')
1547
1566
                ],
1548
1567
                origins)
1549
1568
        else:
1550
1569
            # Without a graph everything is new.
1551
1570
            self.assertEqual([
1552
 
                (prefix + ('merged',), 'base\n'),
1553
 
                (prefix + ('merged',), 'left\n'),
1554
 
                (prefix + ('merged',), 'right\n'),
1555
 
                (prefix + ('merged',), 'merged\n')
 
1571
                (prefix + (b'merged',), b'base\n'),
 
1572
                (prefix + (b'merged',), b'left\n'),
 
1573
                (prefix + (b'merged',), b'right\n'),
 
1574
                (prefix + (b'merged',), b'merged\n')
1556
1575
                ],
1557
1576
                origins)
1558
1577
        self.assertRaises(RevisionNotPresent,
1559
 
            files.annotate, prefix + ('missing-key',))
 
1578
                          files.annotate, prefix + ('missing-key',))
1560
1579
 
1561
1580
    def test_check_no_parameters(self):
1562
1581
        files = self.get_versionedfiles()
1576
1595
        seen = set()
1577
1596
        # Texts output should be fulltexts.
1578
1597
        self.capture_stream(files, entries, seen.add,
1579
 
            files.get_parent_map(keys), require_fulltext=True)
 
1598
                            files.get_parent_map(keys), require_fulltext=True)
1580
1599
        # All texts should be output.
1581
1600
        self.assertEqual(set(keys), seen)
1582
1601
 
1589
1608
        files = self.get_versionedfiles()
1590
1609
 
1591
1610
    def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1592
 
        nokeys=False):
 
1611
                          nokeys=False):
1593
1612
        return get_diamond_files(files, self.key_length,
1594
 
            trailing_eol=trailing_eol, nograph=not self.graph,
1595
 
            left_only=left_only, nokeys=nokeys)
 
1613
                                 trailing_eol=trailing_eol, nograph=not self.graph,
 
1614
                                 left_only=left_only, nokeys=nokeys)
1596
1615
 
1597
1616
    def _add_content_nostoresha(self, add_lines):
1598
1617
        """When nostore_sha is supplied using old content raises."""
1599
1618
        vf = self.get_versionedfiles()
1600
 
        empty_text = ('a', [])
1601
 
        sample_text_nl = ('b', ["foo\n", "bar\n"])
1602
 
        sample_text_no_nl = ('c', ["foo\n", "bar"])
 
1619
        empty_text = (b'a', [])
 
1620
        sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
 
1621
        sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
1603
1622
        shas = []
1604
1623
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1605
1624
            if add_lines:
1606
1625
                sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1607
1626
                                         lines)
1608
1627
            else:
1609
 
                sha, _, _ = vf._add_text(self.get_simple_key(version), [],
1610
 
                                         ''.join(lines))
 
1628
                sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
 
1629
                                         lines)
1611
1630
            shas.append(sha)
1612
1631
        # we now have a copy of all the lines in the vf.
1613
1632
        for sha, (version, lines) in zip(
1614
 
            shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1615
 
            new_key = self.get_simple_key(version + "2")
1616
 
            self.assertRaises(errors.ExistingContent,
1617
 
                vf.add_lines, new_key, [], lines,
1618
 
                nostore_sha=sha)
1619
 
            self.assertRaises(errors.ExistingContent,
1620
 
                vf._add_text, new_key, [], ''.join(lines),
1621
 
                nostore_sha=sha)
 
1633
                shas, (empty_text, sample_text_nl, sample_text_no_nl)):
 
1634
            new_key = self.get_simple_key(version + b"2")
 
1635
            self.assertRaises(errors.ExistingContent,
 
1636
                              vf.add_lines, new_key, [], lines,
 
1637
                              nostore_sha=sha)
 
1638
            self.assertRaises(errors.ExistingContent,
 
1639
                              vf.add_lines, new_key, [], lines,
 
1640
                              nostore_sha=sha)
1622
1641
            # and no new version should have been added.
1623
 
            record = vf.get_record_stream([new_key], 'unordered', True).next()
 
1642
            record = next(vf.get_record_stream([new_key], 'unordered', True))
1624
1643
            self.assertEqual('absent', record.storage_kind)
1625
1644
 
1626
1645
    def test_add_lines_nostoresha(self):
1627
1646
        self._add_content_nostoresha(add_lines=True)
1628
1647
 
1629
 
    def test__add_text_nostoresha(self):
1630
 
        self._add_content_nostoresha(add_lines=False)
1631
 
 
1632
1648
    def test_add_lines_return(self):
1633
1649
        files = self.get_versionedfiles()
1634
1650
        # save code by using the stock data insertion helper.
1640
1656
            results.append(add[:2])
1641
1657
        if self.key_length == 1:
1642
1658
            self.assertEqual([
1643
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1644
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1645
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1646
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1647
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
 
1659
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1660
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1661
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1662
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1663
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1648
1664
                results)
1649
1665
        elif self.key_length == 2:
1650
1666
            self.assertEqual([
1651
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1652
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1653
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1654
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1655
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1656
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1657
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1658
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1659
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1660
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
 
1667
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1668
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1669
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1670
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1671
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1672
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1673
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1674
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1675
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
 
1676
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1661
1677
                results)
1662
1678
 
1663
1679
    def test_add_lines_no_key_generates_chk_key(self):
1671
1687
            results.append(add[:2])
1672
1688
        if self.key_length == 1:
1673
1689
            self.assertEqual([
1674
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1675
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1676
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1677
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1678
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
 
1690
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1691
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1692
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1693
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1694
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1679
1695
                results)
1680
1696
            # Check the added items got CHK keys.
1681
 
            self.assertEqual(set([
1682
 
                ('sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1683
 
                ('sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1684
 
                ('sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1685
 
                ('sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1686
 
                ('sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1687
 
                ]),
 
1697
            self.assertEqual({
 
1698
                (b'sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
 
1699
                (b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
 
1700
                (b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
 
1701
                (b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
 
1702
                (b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
 
1703
                },
1688
1704
                files.keys())
1689
1705
        elif self.key_length == 2:
1690
1706
            self.assertEqual([
1691
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1692
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1693
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1694
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1695
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1696
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1697
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1698
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1699
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1700
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
 
1707
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1708
                (b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
 
1709
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1710
                (b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
 
1711
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1712
                (b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
 
1713
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1714
                (b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
 
1715
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
 
1716
                (b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1701
1717
                results)
1702
1718
            # Check the added items got CHK keys.
1703
 
            self.assertEqual(set([
1704
 
                ('FileA', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1705
 
                ('FileA', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1706
 
                ('FileA', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1707
 
                ('FileA', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1708
 
                ('FileA', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1709
 
                ('FileB', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1710
 
                ('FileB', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1711
 
                ('FileB', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1712
 
                ('FileB', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1713
 
                ('FileB', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1714
 
                ]),
 
1719
            self.assertEqual({
 
1720
                (b'FileA', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
 
1721
                (b'FileA', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
 
1722
                (b'FileA', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
 
1723
                (b'FileA', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
 
1724
                (b'FileA', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
 
1725
                (b'FileB', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
 
1726
                (b'FileB', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
 
1727
                (b'FileB', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
 
1728
                (b'FileB', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
 
1729
                (b'FileB', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
 
1730
                },
1715
1731
                files.keys())
1716
1732
 
1717
1733
    def test_empty_lines(self):
1718
1734
        """Empty files can be stored."""
1719
1735
        f = self.get_versionedfiles()
1720
 
        key_a = self.get_simple_key('a')
 
1736
        key_a = self.get_simple_key(b'a')
1721
1737
        f.add_lines(key_a, [], [])
1722
 
        self.assertEqual('',
1723
 
            f.get_record_stream([key_a], 'unordered', True
1724
 
                ).next().get_bytes_as('fulltext'))
1725
 
        key_b = self.get_simple_key('b')
 
1738
        self.assertEqual(b'',
 
1739
                         next(f.get_record_stream([key_a], 'unordered', True
 
1740
                                                  )).get_bytes_as('fulltext'))
 
1741
        key_b = self.get_simple_key(b'b')
1726
1742
        f.add_lines(key_b, self.get_parents([key_a]), [])
1727
 
        self.assertEqual('',
1728
 
            f.get_record_stream([key_b], 'unordered', True
1729
 
                ).next().get_bytes_as('fulltext'))
 
1743
        self.assertEqual(b'',
 
1744
                         next(f.get_record_stream([key_b], 'unordered', True
 
1745
                                                  )).get_bytes_as('fulltext'))
1730
1746
 
1731
1747
    def test_newline_only(self):
1732
1748
        f = self.get_versionedfiles()
1733
 
        key_a = self.get_simple_key('a')
1734
 
        f.add_lines(key_a, [], ['\n'])
1735
 
        self.assertEqual('\n',
1736
 
            f.get_record_stream([key_a], 'unordered', True
1737
 
                ).next().get_bytes_as('fulltext'))
1738
 
        key_b = self.get_simple_key('b')
1739
 
        f.add_lines(key_b, self.get_parents([key_a]), ['\n'])
1740
 
        self.assertEqual('\n',
1741
 
            f.get_record_stream([key_b], 'unordered', True
1742
 
                ).next().get_bytes_as('fulltext'))
 
1749
        key_a = self.get_simple_key(b'a')
 
1750
        f.add_lines(key_a, [], [b'\n'])
 
1751
        self.assertEqual(b'\n',
 
1752
                         next(f.get_record_stream([key_a], 'unordered', True
 
1753
                                                  )).get_bytes_as('fulltext'))
 
1754
        key_b = self.get_simple_key(b'b')
 
1755
        f.add_lines(key_b, self.get_parents([key_a]), [b'\n'])
 
1756
        self.assertEqual(b'\n',
 
1757
                         next(f.get_record_stream([key_b], 'unordered', True
 
1758
                                                  )).get_bytes_as('fulltext'))
1743
1759
 
1744
1760
    def test_get_known_graph_ancestry(self):
1745
1761
        f = self.get_versionedfiles()
1746
1762
        if not self.graph:
1747
1763
            raise TestNotApplicable('ancestry info only relevant with graph.')
1748
 
        key_a = self.get_simple_key('a')
1749
 
        key_b = self.get_simple_key('b')
1750
 
        key_c = self.get_simple_key('c')
 
1764
        key_a = self.get_simple_key(b'a')
 
1765
        key_b = self.get_simple_key(b'b')
 
1766
        key_c = self.get_simple_key(b'c')
1751
1767
        # A
1752
1768
        # |\
1753
1769
        # | B
1754
1770
        # |/
1755
1771
        # C
1756
 
        f.add_lines(key_a, [], ['\n'])
1757
 
        f.add_lines(key_b, [key_a], ['\n'])
1758
 
        f.add_lines(key_c, [key_a, key_b], ['\n'])
 
1772
        f.add_lines(key_a, [], [b'\n'])
 
1773
        f.add_lines(key_b, [key_a], [b'\n'])
 
1774
        f.add_lines(key_c, [key_a, key_b], [b'\n'])
1759
1775
        kg = f.get_known_graph_ancestry([key_c])
1760
1776
        self.assertIsInstance(kg, _mod_graph.KnownGraph)
1761
1777
        self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1767
1783
        if getattr(f, 'add_fallback_versioned_files', None) is None:
1768
1784
            raise TestNotApplicable("%s doesn't support fallbacks"
1769
1785
                                    % (f.__class__.__name__,))
1770
 
        key_a = self.get_simple_key('a')
1771
 
        key_b = self.get_simple_key('b')
1772
 
        key_c = self.get_simple_key('c')
 
1786
        key_a = self.get_simple_key(b'a')
 
1787
        key_b = self.get_simple_key(b'b')
 
1788
        key_c = self.get_simple_key(b'c')
1773
1789
        # A     only in fallback
1774
1790
        # |\
1775
1791
        # | B
1776
1792
        # |/
1777
1793
        # C
1778
1794
        g = self.get_versionedfiles('fallback')
1779
 
        g.add_lines(key_a, [], ['\n'])
 
1795
        g.add_lines(key_a, [], [b'\n'])
1780
1796
        f.add_fallback_versioned_files(g)
1781
 
        f.add_lines(key_b, [key_a], ['\n'])
1782
 
        f.add_lines(key_c, [key_a, key_b], ['\n'])
 
1797
        f.add_lines(key_b, [key_a], [b'\n'])
 
1798
        f.add_lines(key_c, [key_a, key_b], [b'\n'])
1783
1799
        kg = f.get_known_graph_ancestry([key_c])
1784
1800
        self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1785
1801
 
1792
1808
    def assertValidStorageKind(self, storage_kind):
1793
1809
        """Assert that storage_kind is a valid storage_kind."""
1794
1810
        self.assertSubset([storage_kind],
1795
 
            ['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1796
 
             'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1797
 
             'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1798
 
             'knit-delta-gz',
1799
 
             'knit-delta-closure', 'knit-delta-closure-ref',
1800
 
             'groupcompress-block', 'groupcompress-block-ref'])
 
1811
                          ['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
 
1812
                           'knit-ft', 'knit-delta', 'chunked', 'fulltext',
 
1813
                           'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
 
1814
                           'knit-delta-gz',
 
1815
                           'knit-delta-closure', 'knit-delta-closure-ref',
 
1816
                           'groupcompress-block', 'groupcompress-block-ref'])
1801
1817
 
1802
1818
    def capture_stream(self, f, entries, on_seen, parents,
1803
 
        require_fulltext=False):
 
1819
                       require_fulltext=False):
1804
1820
        """Capture a stream for testing."""
1805
1821
        for factory in entries:
1806
1822
            on_seen(factory.key)
1807
1823
            self.assertValidStorageKind(factory.storage_kind)
1808
1824
            if factory.sha1 is not None:
1809
1825
                self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1810
 
                    factory.sha1)
 
1826
                                 factory.sha1)
1811
1827
            self.assertEqual(parents[factory.key], factory.parents)
1812
1828
            self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1813
 
                str)
 
1829
                                  bytes)
1814
1830
            if require_fulltext:
1815
1831
                factory.get_bytes_as('fulltext')
1816
1832
 
1828
1844
    def get_keys_and_sort_order(self):
1829
1845
        """Get diamond test keys list, and their sort ordering."""
1830
1846
        if self.key_length == 1:
1831
 
            keys = [('merged',), ('left',), ('right',), ('base',)]
1832
 
            sort_order = {('merged',):2, ('left',):1, ('right',):1, ('base',):0}
 
1847
            keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
 
1848
            sort_order = {(b'merged',): 2, (b'left',): 1,
 
1849
                          (b'right',): 1, (b'base',): 0}
1833
1850
        else:
1834
1851
            keys = [
1835
 
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1836
 
                ('FileA', 'base'),
1837
 
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1838
 
                ('FileB', 'base'),
 
1852
                (b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
 
1853
                (b'FileA', b'base'),
 
1854
                (b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
 
1855
                (b'FileB', b'base'),
1839
1856
                ]
1840
1857
            sort_order = {
1841
 
                ('FileA', 'merged'):2, ('FileA', 'left'):1, ('FileA', 'right'):1,
1842
 
                ('FileA', 'base'):0,
1843
 
                ('FileB', 'merged'):2, ('FileB', 'left'):1, ('FileB', 'right'):1,
1844
 
                ('FileB', 'base'):0,
 
1858
                (b'FileA', b'merged'): 2, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
 
1859
                (b'FileA', b'base'): 0,
 
1860
                (b'FileB', b'merged'): 2, (b'FileB', b'left'): 1, (b'FileB', b'right'): 1,
 
1861
                (b'FileB', b'base'): 0,
1845
1862
                }
1846
1863
        return keys, sort_order
1847
1864
 
1848
1865
    def get_keys_and_groupcompress_sort_order(self):
1849
1866
        """Get diamond test keys list, and their groupcompress sort ordering."""
1850
1867
        if self.key_length == 1:
1851
 
            keys = [('merged',), ('left',), ('right',), ('base',)]
1852
 
            sort_order = {('merged',):0, ('left',):1, ('right',):1, ('base',):2}
 
1868
            keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
 
1869
            sort_order = {(b'merged',): 0, (b'left',): 1,
 
1870
                          (b'right',): 1, (b'base',): 2}
1853
1871
        else:
1854
1872
            keys = [
1855
 
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1856
 
                ('FileA', 'base'),
1857
 
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1858
 
                ('FileB', 'base'),
 
1873
                (b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
 
1874
                (b'FileA', b'base'),
 
1875
                (b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
 
1876
                (b'FileB', b'base'),
1859
1877
                ]
1860
1878
            sort_order = {
1861
 
                ('FileA', 'merged'):0, ('FileA', 'left'):1, ('FileA', 'right'):1,
1862
 
                ('FileA', 'base'):2,
1863
 
                ('FileB', 'merged'):3, ('FileB', 'left'):4, ('FileB', 'right'):4,
1864
 
                ('FileB', 'base'):5,
 
1879
                (b'FileA', b'merged'): 0, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
 
1880
                (b'FileA', b'base'): 2,
 
1881
                (b'FileB', b'merged'): 3, (b'FileB', b'left'): 4, (b'FileB', b'right'): 4,
 
1882
                (b'FileB', b'base'): 5,
1865
1883
                }
1866
1884
        return keys, sort_order
1867
1885
 
1888
1906
            seen.append(factory.key)
1889
1907
            self.assertValidStorageKind(factory.storage_kind)
1890
1908
            self.assertSubset([factory.sha1],
1891
 
                [None, files.get_sha1s([factory.key])[factory.key]])
 
1909
                              [None, files.get_sha1s([factory.key])[factory.key]])
1892
1910
            self.assertEqual(parent_map[factory.key], factory.parents)
1893
1911
            # self.assertEqual(files.get_text(factory.key),
1894
1912
            ft_bytes = factory.get_bytes_as('fulltext')
1895
 
            self.assertIsInstance(ft_bytes, str)
 
1913
            self.assertIsInstance(ft_bytes, bytes)
1896
1914
            chunked_bytes = factory.get_bytes_as('chunked')
1897
 
            self.assertEqualDiff(ft_bytes, ''.join(chunked_bytes))
 
1915
            self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1898
1916
 
1899
1917
        self.assertStreamOrder(sort_order, seen, keys)
1900
1918
 
1912
1930
    def assertStreamOrder(self, sort_order, seen, keys):
1913
1931
        self.assertEqual(len(set(seen)), len(keys))
1914
1932
        if self.key_length == 1:
1915
 
            lows = {():0}
 
1933
            lows = {(): 0}
1916
1934
        else:
1917
 
            lows = {('FileA',):0, ('FileB',):0}
 
1935
            lows = {(b'FileA',): 0, (b'FileB',): 0}
1918
1936
        if not self.graph:
1919
1937
            self.assertEqual(set(keys), set(seen))
1920
1938
        else:
1921
1939
            for key in seen:
1922
1940
                sort_pos = sort_order[key]
1923
1941
                self.assertTrue(sort_pos >= lows[key[:-1]],
1924
 
                    "Out of order in sorted stream: %r, %r" % (key, seen))
 
1942
                                "Out of order in sorted stream: %r, %r" % (key, seen))
1925
1943
                lows[key[:-1]] = sort_pos
1926
1944
 
1927
1945
    def test_get_record_stream_unknown_storage_kind_raises(self):
1929
1947
        files = self.get_versionedfiles()
1930
1948
        self.get_diamond_files(files)
1931
1949
        if self.key_length == 1:
1932
 
            keys = [('merged',), ('left',), ('right',), ('base',)]
 
1950
            keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1933
1951
        else:
1934
1952
            keys = [
1935
 
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1936
 
                ('FileA', 'base'),
1937
 
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1938
 
                ('FileB', 'base'),
 
1953
                (b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
 
1954
                (b'FileA', b'base'),
 
1955
                (b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
 
1956
                (b'FileB', b'base'),
1939
1957
                ]
1940
1958
        parent_map = files.get_parent_map(keys)
1941
1959
        entries = files.get_record_stream(keys, 'unordered', False)
1951
1969
            self.assertEqual(parent_map[factory.key], factory.parents)
1952
1970
            # currently no stream emits mpdiff
1953
1971
            self.assertRaises(errors.UnavailableRepresentation,
1954
 
                factory.get_bytes_as, 'mpdiff')
 
1972
                              factory.get_bytes_as, 'mpdiff')
1955
1973
            self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1956
 
                str)
 
1974
                                  bytes)
1957
1975
        self.assertEqual(set(keys), seen)
1958
1976
 
1959
1977
    def test_get_record_stream_missing_records_are_absent(self):
1960
1978
        files = self.get_versionedfiles()
1961
1979
        self.get_diamond_files(files)
1962
1980
        if self.key_length == 1:
1963
 
            keys = [('merged',), ('left',), ('right',), ('absent',), ('base',)]
 
1981
            keys = [(b'merged',), (b'left',), (b'right',),
 
1982
                    (b'absent',), (b'base',)]
1964
1983
        else:
1965
1984
            keys = [
1966
 
                ('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1967
 
                ('FileA', 'absent'), ('FileA', 'base'),
1968
 
                ('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1969
 
                ('FileB', 'absent'), ('FileB', 'base'),
1970
 
                ('absent', 'absent'),
 
1985
                (b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
 
1986
                (b'FileA', b'absent'), (b'FileA', b'base'),
 
1987
                (b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
 
1988
                (b'FileB', b'absent'), (b'FileB', b'base'),
 
1989
                (b'absent', b'absent'),
1971
1990
                ]
1972
1991
        parent_map = files.get_parent_map(keys)
1973
1992
        entries = files.get_record_stream(keys, 'unordered', False)
1978
1997
    def assertRecordHasContent(self, record, bytes):
1979
1998
        """Assert that record has the bytes bytes."""
1980
1999
        self.assertEqual(bytes, record.get_bytes_as('fulltext'))
1981
 
        self.assertEqual(bytes, ''.join(record.get_bytes_as('chunked')))
 
2000
        self.assertEqual(bytes, b''.join(record.get_bytes_as('chunked')))
1982
2001
 
1983
2002
    def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
1984
2003
        files = self.get_versionedfiles()
1985
 
        key = self.get_simple_key('foo')
1986
 
        files.add_lines(key, (), ['my text\n', 'content'])
 
2004
        key = self.get_simple_key(b'foo')
 
2005
        files.add_lines(key, (), [b'my text\n', b'content'])
1987
2006
        stream = files.get_record_stream([key], 'unordered', False)
1988
 
        record = stream.next()
 
2007
        record = next(stream)
1989
2008
        if record.storage_kind in ('chunked', 'fulltext'):
1990
2009
            # chunked and fulltext representations are for direct use not wire
1991
2010
            # serialisation: check they are able to be used directly. To send
1992
2011
            # such records over the wire translation will be needed.
1993
 
            self.assertRecordHasContent(record, "my text\ncontent")
 
2012
            self.assertRecordHasContent(record, b"my text\ncontent")
1994
2013
        else:
1995
2014
            bytes = [record.get_bytes_as(record.storage_kind)]
1996
2015
            network_stream = versionedfile.NetworkRecordStream(bytes).read()
1999
2018
            for record in network_stream:
2000
2019
                records.append(record)
2001
2020
                self.assertEqual(source_record.storage_kind,
2002
 
                    record.storage_kind)
 
2021
                                 record.storage_kind)
2003
2022
                self.assertEqual(source_record.parents, record.parents)
2004
2023
                self.assertEqual(
2005
2024
                    source_record.get_bytes_as(source_record.storage_kind),
2012
2031
        :param records: A list to collect the seen records.
2013
2032
        :return: A generator of the records in stream.
2014
2033
        """
2015
 
        # We make assertions during copying to catch things early for
2016
 
        # easier debugging.
2017
 
        for record, ref_record in izip(stream, expected):
 
2034
        # We make assertions during copying to catch things early for easier
 
2035
        # debugging. This must use the iterating zip() from the future.
 
2036
        for record, ref_record in zip(stream, expected):
2018
2037
            records.append(record)
2019
2038
            self.assertEqual(ref_record.key, record.key)
2020
2039
            self.assertEqual(ref_record.storage_kind, record.storage_kind)
2022
2041
            yield record
2023
2042
 
2024
2043
    def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
2025
 
        stream):
 
2044
                                        stream):
2026
2045
        """Convert a stream to a bytes iterator.
2027
2046
 
2028
2047
        :param skipped_records: A list with one element to increment when a
2043
2062
    def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
2044
2063
        files = self.get_versionedfiles()
2045
2064
        target_files = self.get_versionedfiles('target')
2046
 
        key = self.get_simple_key('ft')
2047
 
        key_delta = self.get_simple_key('delta')
2048
 
        files.add_lines(key, (), ['my text\n', 'content'])
 
2065
        key = self.get_simple_key(b'ft')
 
2066
        key_delta = self.get_simple_key(b'delta')
 
2067
        files.add_lines(key, (), [b'my text\n', b'content'])
2049
2068
        if self.graph:
2050
2069
            delta_parents = (key,)
2051
2070
        else:
2052
2071
            delta_parents = ()
2053
 
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
 
2072
        files.add_lines(key_delta, delta_parents, [
 
2073
                        b'different\n', b'content\n'])
2054
2074
        local = files.get_record_stream([key, key_delta], 'unordered', False)
2055
2075
        ref = files.get_record_stream([key, key_delta], 'unordered', False)
2056
2076
        skipped_records = [0]
2057
2077
        full_texts = {
2058
 
            key: "my text\ncontent",
2059
 
            key_delta: "different\ncontent\n",
 
2078
            key: b"my text\ncontent",
 
2079
            key_delta: b"different\ncontent\n",
2060
2080
            }
2061
2081
        byte_stream = self.stream_to_bytes_or_skip_counter(
2062
2082
            skipped_records, full_texts, local)
2077
2097
        # copy a delta over the wire
2078
2098
        files = self.get_versionedfiles()
2079
2099
        target_files = self.get_versionedfiles('target')
2080
 
        key = self.get_simple_key('ft')
2081
 
        key_delta = self.get_simple_key('delta')
2082
 
        files.add_lines(key, (), ['my text\n', 'content'])
 
2100
        key = self.get_simple_key(b'ft')
 
2101
        key_delta = self.get_simple_key(b'delta')
 
2102
        files.add_lines(key, (), [b'my text\n', b'content'])
2083
2103
        if self.graph:
2084
2104
            delta_parents = (key,)
2085
2105
        else:
2086
2106
            delta_parents = ()
2087
 
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
 
2107
        files.add_lines(key_delta, delta_parents, [
 
2108
                        b'different\n', b'content\n'])
2088
2109
        # Copy the basis text across so we can reconstruct the delta during
2089
2110
        # insertion into target.
2090
2111
        target_files.insert_record_stream(files.get_record_stream([key],
2091
 
            'unordered', False))
 
2112
                                                                  'unordered', False))
2092
2113
        local = files.get_record_stream([key_delta], 'unordered', False)
2093
2114
        ref = files.get_record_stream([key_delta], 'unordered', False)
2094
2115
        skipped_records = [0]
2095
2116
        full_texts = {
2096
 
            key_delta: "different\ncontent\n",
 
2117
            key_delta: b"different\ncontent\n",
2097
2118
            }
2098
2119
        byte_stream = self.stream_to_bytes_or_skip_counter(
2099
2120
            skipped_records, full_texts, local)
2113
2134
    def test_get_record_stream_wire_ready_delta_closure_included(self):
2114
2135
        # copy a delta over the wire with the ability to get its full text.
2115
2136
        files = self.get_versionedfiles()
2116
 
        key = self.get_simple_key('ft')
2117
 
        key_delta = self.get_simple_key('delta')
2118
 
        files.add_lines(key, (), ['my text\n', 'content'])
 
2137
        key = self.get_simple_key(b'ft')
 
2138
        key_delta = self.get_simple_key(b'delta')
 
2139
        files.add_lines(key, (), [b'my text\n', b'content'])
2119
2140
        if self.graph:
2120
2141
            delta_parents = (key,)
2121
2142
        else:
2122
2143
            delta_parents = ()
2123
 
        files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
 
2144
        files.add_lines(key_delta, delta_parents, [
 
2145
                        b'different\n', b'content\n'])
2124
2146
        local = files.get_record_stream([key_delta], 'unordered', True)
2125
2147
        ref = files.get_record_stream([key_delta], 'unordered', True)
2126
2148
        skipped_records = [0]
2127
2149
        full_texts = {
2128
 
            key_delta: "different\ncontent\n",
 
2150
            key_delta: b"different\ncontent\n",
2129
2151
            }
2130
2152
        byte_stream = self.stream_to_bytes_or_skip_counter(
2131
2153
            skipped_records, full_texts, local)
2145
2167
        seen = set()
2146
2168
        for factory in entries:
2147
2169
            seen.add(factory.key)
2148
 
            if factory.key[-1] == 'absent':
 
2170
            if factory.key[-1] == b'absent':
2149
2171
                self.assertEqual('absent', factory.storage_kind)
2150
2172
                self.assertEqual(None, factory.sha1)
2151
2173
                self.assertEqual(None, factory.parents)
2156
2178
                    self.assertEqual(sha1, factory.sha1)
2157
2179
                self.assertEqual(parents[factory.key], factory.parents)
2158
2180
                self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2159
 
                    str)
 
2181
                                      bytes)
2160
2182
        self.assertEqual(set(keys), seen)
2161
2183
 
2162
2184
    def test_filter_absent_records(self):
2170
2192
        # absent keys is still delivered).
2171
2193
        present_keys = list(keys)
2172
2194
        if self.key_length == 1:
2173
 
            keys.insert(2, ('extra',))
 
2195
            keys.insert(2, (b'extra',))
2174
2196
        else:
2175
 
            keys.insert(2, ('extra', 'extra'))
 
2197
            keys.insert(2, (b'extra', b'extra'))
2176
2198
        entries = files.get_record_stream(keys, 'unordered', False)
2177
2199
        seen = set()
2178
2200
        self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2179
 
            parent_map)
 
2201
                            parent_map)
2180
2202
        self.assertEqual(set(present_keys), seen)
2181
2203
 
2182
2204
    def get_mapper(self):
2196
2218
    def test_get_annotator(self):
2197
2219
        files = self.get_versionedfiles()
2198
2220
        self.get_diamond_files(files)
2199
 
        origin_key = self.get_simple_key('origin')
2200
 
        base_key = self.get_simple_key('base')
2201
 
        left_key = self.get_simple_key('left')
2202
 
        right_key = self.get_simple_key('right')
2203
 
        merged_key = self.get_simple_key('merged')
 
2221
        origin_key = self.get_simple_key(b'origin')
 
2222
        base_key = self.get_simple_key(b'base')
 
2223
        left_key = self.get_simple_key(b'left')
 
2224
        right_key = self.get_simple_key(b'right')
 
2225
        merged_key = self.get_simple_key(b'merged')
2204
2226
        # annotator = files.get_annotator()
2205
2227
        # introduced full text
2206
2228
        origins, lines = files.get_annotator().annotate(origin_key)
2207
2229
        self.assertEqual([(origin_key,)], origins)
2208
 
        self.assertEqual(['origin\n'], lines)
 
2230
        self.assertEqual([b'origin\n'], lines)
2209
2231
        # a delta
2210
2232
        origins, lines = files.get_annotator().annotate(base_key)
2211
2233
        self.assertEqual([(base_key,)], origins)
2227
2249
                (merged_key,),
2228
2250
                ], origins)
2229
2251
        self.assertRaises(RevisionNotPresent,
2230
 
            files.get_annotator().annotate, self.get_simple_key('missing-key'))
 
2252
                          files.get_annotator().annotate, self.get_simple_key(b'missing-key'))
2231
2253
 
2232
2254
    def test_get_parent_map(self):
2233
2255
        files = self.get_versionedfiles()
2234
2256
        if self.key_length == 1:
2235
2257
            parent_details = [
2236
 
                (('r0',), self.get_parents(())),
2237
 
                (('r1',), self.get_parents((('r0',),))),
2238
 
                (('r2',), self.get_parents(())),
2239
 
                (('r3',), self.get_parents(())),
2240
 
                (('m',), self.get_parents((('r0',),('r1',),('r2',),('r3',)))),
 
2258
                ((b'r0',), self.get_parents(())),
 
2259
                ((b'r1',), self.get_parents(((b'r0',),))),
 
2260
                ((b'r2',), self.get_parents(())),
 
2261
                ((b'r3',), self.get_parents(())),
 
2262
                ((b'm',), self.get_parents(((b'r0',), (b'r1',), (b'r2',), (b'r3',)))),
2241
2263
                ]
2242
2264
        else:
2243
2265
            parent_details = [
2244
 
                (('FileA', 'r0'), self.get_parents(())),
2245
 
                (('FileA', 'r1'), self.get_parents((('FileA', 'r0'),))),
2246
 
                (('FileA', 'r2'), self.get_parents(())),
2247
 
                (('FileA', 'r3'), self.get_parents(())),
2248
 
                (('FileA', 'm'), self.get_parents((('FileA', 'r0'),
2249
 
                    ('FileA', 'r1'), ('FileA', 'r2'), ('FileA', 'r3')))),
 
2266
                ((b'FileA', b'r0'), self.get_parents(())),
 
2267
                ((b'FileA', b'r1'), self.get_parents(((b'FileA', b'r0'),))),
 
2268
                ((b'FileA', b'r2'), self.get_parents(())),
 
2269
                ((b'FileA', b'r3'), self.get_parents(())),
 
2270
                ((b'FileA', b'm'), self.get_parents(((b'FileA', b'r0'),
 
2271
                                                     (b'FileA', b'r1'), (b'FileA', b'r2'), (b'FileA', b'r3')))),
2250
2272
                ]
2251
2273
        for key, parents in parent_details:
2252
2274
            files.add_lines(key, parents, [])
2253
2275
            # immediately after adding it should be queryable.
2254
 
            self.assertEqual({key:parents}, files.get_parent_map([key]))
 
2276
            self.assertEqual({key: parents}, files.get_parent_map([key]))
2255
2277
        # We can ask for an empty set
2256
2278
        self.assertEqual({}, files.get_parent_map([]))
2257
2279
        # We can ask for many keys
2258
2280
        all_parents = dict(parent_details)
2259
2281
        self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2260
2282
        # Absent keys are just not included in the result.
2261
 
        keys = all_parents.keys()
 
2283
        keys = list(all_parents.keys())
2262
2284
        if self.key_length == 1:
2263
 
            keys.insert(1, ('missing',))
 
2285
            keys.insert(1, (b'missing',))
2264
2286
        else:
2265
 
            keys.insert(1, ('missing', 'missing'))
 
2287
            keys.insert(1, (b'missing', b'missing'))
2266
2288
        # Absent keys are just ignored
2267
2289
        self.assertEqual(all_parents, files.get_parent_map(keys))
2268
2290
 
2270
2292
        files = self.get_versionedfiles()
2271
2293
        self.get_diamond_files(files)
2272
2294
        if self.key_length == 1:
2273
 
            keys = [('base',), ('origin',), ('left',), ('merged',), ('right',)]
 
2295
            keys = [(b'base',), (b'origin',), (b'left',),
 
2296
                    (b'merged',), (b'right',)]
2274
2297
        else:
2275
2298
            # ask for shas from different prefixes.
2276
2299
            keys = [
2277
 
                ('FileA', 'base'), ('FileB', 'origin'), ('FileA', 'left'),
2278
 
                ('FileA', 'merged'), ('FileB', 'right'),
 
2300
                (b'FileA', b'base'), (b'FileB', b'origin'), (b'FileA', b'left'),
 
2301
                (b'FileA', b'merged'), (b'FileB', b'right'),
2279
2302
                ]
2280
2303
        self.assertEqual({
2281
 
            keys[0]: '51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2282
 
            keys[1]: '00e364d235126be43292ab09cb4686cf703ddc17',
2283
 
            keys[2]: 'a8478686da38e370e32e42e8a0c220e33ee9132f',
2284
 
            keys[3]: 'ed8bce375198ea62444dc71952b22cfc2b09226d',
2285
 
            keys[4]: '9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
 
2304
            keys[0]: b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
 
2305
            keys[1]: b'00e364d235126be43292ab09cb4686cf703ddc17',
 
2306
            keys[2]: b'a8478686da38e370e32e42e8a0c220e33ee9132f',
 
2307
            keys[3]: b'ed8bce375198ea62444dc71952b22cfc2b09226d',
 
2308
            keys[4]: b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2286
2309
            },
2287
2310
            files.get_sha1s(keys))
2288
2311
 
2296
2319
        self.assertEqual(set(actual.keys()), set(expected.keys()))
2297
2320
        actual_parents = actual.get_parent_map(actual.keys())
2298
2321
        if self.graph:
2299
 
            self.assertEqual(actual_parents, expected.get_parent_map(expected.keys()))
 
2322
            self.assertEqual(
 
2323
                actual_parents, expected.get_parent_map(expected.keys()))
2300
2324
        else:
2301
2325
            for key, parents in actual_parents.items():
2302
2326
                self.assertEqual(None, parents)
2303
2327
        for key in actual.keys():
2304
 
            actual_text = actual.get_record_stream(
2305
 
                [key], 'unordered', True).next().get_bytes_as('fulltext')
2306
 
            expected_text = expected.get_record_stream(
2307
 
                [key], 'unordered', True).next().get_bytes_as('fulltext')
 
2328
            actual_text = next(actual.get_record_stream(
 
2329
                [key], 'unordered', True)).get_bytes_as('fulltext')
 
2330
            expected_text = next(expected.get_record_stream(
 
2331
                [key], 'unordered', True)).get_bytes_as('fulltext')
2308
2332
            self.assertEqual(actual_text, expected_text)
2309
2333
 
2310
2334
    def test_insert_record_stream_fulltexts(self):
2318
2342
            source_transport)
2319
2343
        self.get_diamond_files(source, trailing_eol=False)
2320
2344
        stream = source.get_record_stream(source.keys(), 'topological',
2321
 
            False)
 
2345
                                          False)
2322
2346
        files.insert_record_stream(stream)
2323
2347
        self.assertIdenticalVersionedFile(source, files)
2324
2348
 
2333
2357
            source_transport)
2334
2358
        self.get_diamond_files(source, trailing_eol=False)
2335
2359
        stream = source.get_record_stream(source.keys(), 'topological',
2336
 
            False)
 
2360
                                          False)
2337
2361
        files.insert_record_stream(stream)
2338
2362
        self.assertIdenticalVersionedFile(source, files)
2339
2363
 
2346
2370
        source = make_file_factory(True, mapper)(source_transport)
2347
2371
        self.get_diamond_files(source)
2348
2372
        stream = source.get_record_stream(source.keys(), 'topological',
2349
 
            False)
 
2373
                                          False)
2350
2374
        files.insert_record_stream(stream)
2351
2375
        self.assertIdenticalVersionedFile(source, files)
2352
2376
 
2359
2383
        source = make_file_factory(True, mapper)(source_transport)
2360
2384
        self.get_diamond_files(source, trailing_eol=False)
2361
2385
        stream = source.get_record_stream(source.keys(), 'topological',
2362
 
            False)
 
2386
                                          False)
2363
2387
        files.insert_record_stream(stream)
2364
2388
        self.assertIdenticalVersionedFile(source, files)
2365
2389
 
2372
2396
        source = make_file_factory(False, mapper)(source_transport)
2373
2397
        self.get_diamond_files(source)
2374
2398
        stream = source.get_record_stream(source.keys(), 'topological',
2375
 
            False)
 
2399
                                          False)
2376
2400
        files.insert_record_stream(stream)
2377
2401
        self.assertIdenticalVersionedFile(source, files)
2378
2402
 
2385
2409
        source = make_file_factory(False, mapper)(source_transport)
2386
2410
        self.get_diamond_files(source, trailing_eol=False)
2387
2411
        stream = source.get_record_stream(source.keys(), 'topological',
2388
 
            False)
 
2412
                                          False)
2389
2413
        files.insert_record_stream(stream)
2390
2414
        self.assertIdenticalVersionedFile(source, files)
2391
2415
 
2397
2421
        # insert some keys into f.
2398
2422
        self.get_diamond_files(files, left_only=True)
2399
2423
        stream = source.get_record_stream(source.keys(), 'topological',
2400
 
            False)
 
2424
                                          False)
2401
2425
        files.insert_record_stream(stream)
2402
2426
        self.assertIdenticalVersionedFile(source, files)
2403
2427
 
2405
2429
        """Inserting a stream with absent keys should raise an error."""
2406
2430
        files = self.get_versionedfiles()
2407
2431
        source = self.get_versionedfiles('source')
2408
 
        stream = source.get_record_stream([('missing',) * self.key_length],
2409
 
            'topological', False)
 
2432
        stream = source.get_record_stream([(b'missing',) * self.key_length],
 
2433
                                          'topological', False)
2410
2434
        self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2411
 
            stream)
 
2435
                          stream)
2412
2436
 
2413
2437
    def test_insert_record_stream_out_of_order(self):
2414
2438
        """An out of order stream can either error or work."""
2416
2440
        source = self.get_versionedfiles('source')
2417
2441
        self.get_diamond_files(source)
2418
2442
        if self.key_length == 1:
2419
 
            origin_keys = [('origin',)]
2420
 
            end_keys = [('merged',), ('left',)]
2421
 
            start_keys = [('right',), ('base',)]
 
2443
            origin_keys = [(b'origin',)]
 
2444
            end_keys = [(b'merged',), (b'left',)]
 
2445
            start_keys = [(b'right',), (b'base',)]
2422
2446
        else:
2423
 
            origin_keys = [('FileA', 'origin'), ('FileB', 'origin')]
2424
 
            end_keys = [('FileA', 'merged',), ('FileA', 'left',),
2425
 
                ('FileB', 'merged',), ('FileB', 'left',)]
2426
 
            start_keys = [('FileA', 'right',), ('FileA', 'base',),
2427
 
                ('FileB', 'right',), ('FileB', 'base',)]
2428
 
        origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
 
2447
            origin_keys = [(b'FileA', b'origin'), (b'FileB', b'origin')]
 
2448
            end_keys = [(b'FileA', b'merged',), (b'FileA', b'left',),
 
2449
                        (b'FileB', b'merged',), (b'FileB', b'left',)]
 
2450
            start_keys = [(b'FileA', b'right',), (b'FileA', b'base',),
 
2451
                          (b'FileB', b'right',), (b'FileB', b'base',)]
 
2452
        origin_entries = source.get_record_stream(
 
2453
            origin_keys, 'unordered', False)
2429
2454
        end_entries = source.get_record_stream(end_keys, 'topological', False)
2430
 
        start_entries = source.get_record_stream(start_keys, 'topological', False)
2431
 
        entries = chain(origin_entries, end_entries, start_entries)
 
2455
        start_entries = source.get_record_stream(
 
2456
            start_keys, 'topological', False)
 
2457
        entries = itertools.chain(origin_entries, end_entries, start_entries)
2432
2458
        try:
2433
2459
            files.insert_record_stream(entries)
2434
2460
        except RevisionNotPresent:
2446
2472
        source = self.get_versionedfiles('source')
2447
2473
        parents = ()
2448
2474
        keys = []
2449
 
        content = [('same same %d\n' % n) for n in range(500)]
2450
 
        for letter in 'abcdefghijklmnopqrstuvwxyz':
2451
 
            key = ('key-' + letter,)
 
2475
        content = [(b'same same %d\n' % n) for n in range(500)]
 
2476
        letters = b'abcdefghijklmnopqrstuvwxyz'
 
2477
        for i in range(len(letters)):
 
2478
            letter = letters[i:i + 1]
 
2479
            key = (b'key-' + letter,)
2452
2480
            if self.key_length == 2:
2453
 
                key = ('prefix',) + key
2454
 
            content.append('content for ' + letter + '\n')
 
2481
                key = (b'prefix',) + key
 
2482
            content.append(b'content for ' + letter + b'\n')
2455
2483
            source.add_lines(key, parents, content)
2456
2484
            keys.append(key)
2457
2485
            parents = (key,)
2460
2488
        streams = []
2461
2489
        for key in reversed(keys):
2462
2490
            streams.append(source.get_record_stream([key], 'unordered', False))
2463
 
        deltas = chain(*streams[:-1])
 
2491
        deltas = itertools.chain.from_iterable(streams[:-1])
2464
2492
        files = self.get_versionedfiles()
2465
2493
        try:
2466
2494
            files.insert_record_stream(deltas)
2483
2511
        source_transport.mkdir('.')
2484
2512
        source = make_file_factory(False, mapper)(source_transport)
2485
2513
        get_diamond_files(source, self.key_length, trailing_eol=True,
2486
 
            nograph=False, left_only=False)
 
2514
                          nograph=False, left_only=False)
2487
2515
        return source
2488
2516
 
2489
2517
    def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2492
2520
        not added.
2493
2521
        """
2494
2522
        source = self.get_knit_delta_source()
2495
 
        keys = [self.get_simple_key('origin'), self.get_simple_key('merged')]
 
2523
        keys = [self.get_simple_key(b'origin'), self.get_simple_key(b'merged')]
2496
2524
        entries = source.get_record_stream(keys, 'unordered', False)
2497
2525
        files = self.get_versionedfiles()
2498
2526
        if self.support_partial_insertion:
2499
2527
            self.assertEqual([],
2500
 
                list(files.get_missing_compression_parent_keys()))
 
2528
                             list(files.get_missing_compression_parent_keys()))
2501
2529
            files.insert_record_stream(entries)
2502
2530
            missing_bases = files.get_missing_compression_parent_keys()
2503
 
            self.assertEqual(set([self.get_simple_key('left')]),
2504
 
                set(missing_bases))
 
2531
            self.assertEqual({self.get_simple_key(b'left')},
 
2532
                             set(missing_bases))
2505
2533
            self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2506
2534
        else:
2507
2535
            self.assertRaises(
2519
2547
            raise TestNotApplicable(
2520
2548
                'versioned file scenario does not support partial insertion')
2521
2549
        source = self.get_knit_delta_source()
2522
 
        entries = source.get_record_stream([self.get_simple_key('origin'),
2523
 
            self.get_simple_key('merged')], 'unordered', False)
 
2550
        entries = source.get_record_stream([self.get_simple_key(b'origin'),
 
2551
                                            self.get_simple_key(b'merged')], 'unordered', False)
2524
2552
        files = self.get_versionedfiles()
2525
2553
        files.insert_record_stream(entries)
2526
2554
        missing_bases = files.get_missing_compression_parent_keys()
2527
 
        self.assertEqual(set([self.get_simple_key('left')]),
2528
 
            set(missing_bases))
 
2555
        self.assertEqual({self.get_simple_key(b'left')},
 
2556
                         set(missing_bases))
2529
2557
        # 'merged' is inserted (although a commit of a write group involving
2530
2558
        # this versionedfiles would fail).
2531
 
        merged_key = self.get_simple_key('merged')
 
2559
        merged_key = self.get_simple_key(b'merged')
2532
2560
        self.assertEqual(
2533
 
            [merged_key], files.get_parent_map([merged_key]).keys())
 
2561
            [merged_key], list(files.get_parent_map([merged_key]).keys()))
2534
2562
        # Add the full delta closure of the missing records
2535
2563
        missing_entries = source.get_record_stream(
2536
2564
            missing_bases, 'unordered', True)
2538
2566
        # Now 'merged' is fully inserted (and a commit would succeed).
2539
2567
        self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2540
2568
        self.assertEqual(
2541
 
            [merged_key], files.get_parent_map([merged_key]).keys())
 
2569
            [merged_key], list(files.get_parent_map([merged_key]).keys()))
2542
2570
        files.check()
2543
2571
 
2544
2572
    def test_iter_lines_added_or_present_in_keys(self):
2558
2586
 
2559
2587
        files = self.get_versionedfiles()
2560
2588
        # add a base to get included
2561
 
        files.add_lines(self.get_simple_key('base'), (), ['base\n'])
 
2589
        files.add_lines(self.get_simple_key(b'base'), (), [b'base\n'])
2562
2590
        # add a ancestor to be included on one side
2563
 
        files.add_lines(self.get_simple_key('lancestor'), (), ['lancestor\n'])
 
2591
        files.add_lines(self.get_simple_key(
 
2592
            b'lancestor'), (), [b'lancestor\n'])
2564
2593
        # add a ancestor to be included on the other side
2565
 
        files.add_lines(self.get_simple_key('rancestor'),
2566
 
            self.get_parents([self.get_simple_key('base')]), ['rancestor\n'])
 
2594
        files.add_lines(self.get_simple_key(b'rancestor'),
 
2595
                        self.get_parents([self.get_simple_key(b'base')]), [b'rancestor\n'])
2567
2596
        # add a child of rancestor with no eofile-nl
2568
 
        files.add_lines(self.get_simple_key('child'),
2569
 
            self.get_parents([self.get_simple_key('rancestor')]),
2570
 
            ['base\n', 'child\n'])
 
2597
        files.add_lines(self.get_simple_key(b'child'),
 
2598
                        self.get_parents([self.get_simple_key(b'rancestor')]),
 
2599
                        [b'base\n', b'child\n'])
2571
2600
        # add a child of lancestor and base to join the two roots
2572
 
        files.add_lines(self.get_simple_key('otherchild'),
2573
 
            self.get_parents([self.get_simple_key('lancestor'),
2574
 
                self.get_simple_key('base')]),
2575
 
            ['base\n', 'lancestor\n', 'otherchild\n'])
 
2601
        files.add_lines(self.get_simple_key(b'otherchild'),
 
2602
                        self.get_parents([self.get_simple_key(b'lancestor'),
 
2603
                                          self.get_simple_key(b'base')]),
 
2604
                        [b'base\n', b'lancestor\n', b'otherchild\n'])
 
2605
 
2576
2606
        def iter_with_keys(keys, expected):
2577
2607
            # now we need to see what lines are returned, and how often.
2578
2608
            lines = {}
2579
2609
            progress = InstrumentedProgress()
2580
2610
            # iterate over the lines
2581
2611
            for line in files.iter_lines_added_or_present_in_keys(keys,
2582
 
                pb=progress):
 
2612
                                                                  pb=progress):
2583
2613
                lines.setdefault(line, 0)
2584
2614
                lines[line] += 1
2585
 
            if []!= progress.updates:
 
2615
            if [] != progress.updates:
2586
2616
                self.assertEqual(expected, progress.updates)
2587
2617
            return lines
2588
2618
        lines = iter_with_keys(
2589
 
            [self.get_simple_key('child'), self.get_simple_key('otherchild')],
 
2619
            [self.get_simple_key(b'child'),
 
2620
             self.get_simple_key(b'otherchild')],
2590
2621
            [('Walking content', 0, 2),
2591
2622
             ('Walking content', 1, 2),
2592
2623
             ('Walking content', 2, 2)])
2593
2624
        # we must see child and otherchild
2594
 
        self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
 
2625
        self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2595
2626
        self.assertTrue(
2596
 
            lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
 
2627
            lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2597
2628
        # we dont care if we got more than that.
2598
2629
 
2599
2630
        # test all lines
2600
2631
        lines = iter_with_keys(files.keys(),
2601
 
            [('Walking content', 0, 5),
2602
 
             ('Walking content', 1, 5),
2603
 
             ('Walking content', 2, 5),
2604
 
             ('Walking content', 3, 5),
2605
 
             ('Walking content', 4, 5),
2606
 
             ('Walking content', 5, 5)])
 
2632
                               [('Walking content', 0, 5),
 
2633
                                ('Walking content', 1, 5),
 
2634
                                ('Walking content', 2, 5),
 
2635
                                ('Walking content', 3, 5),
 
2636
                                ('Walking content', 4, 5),
 
2637
                                ('Walking content', 5, 5)])
2607
2638
        # all lines must be seen at least once
2608
 
        self.assertTrue(lines[('base\n', self.get_simple_key('base'))] > 0)
2609
 
        self.assertTrue(
2610
 
            lines[('lancestor\n', self.get_simple_key('lancestor'))] > 0)
2611
 
        self.assertTrue(
2612
 
            lines[('rancestor\n', self.get_simple_key('rancestor'))] > 0)
2613
 
        self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2614
 
        self.assertTrue(
2615
 
            lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
 
2639
        self.assertTrue(lines[(b'base\n', self.get_simple_key(b'base'))] > 0)
 
2640
        self.assertTrue(
 
2641
            lines[(b'lancestor\n', self.get_simple_key(b'lancestor'))] > 0)
 
2642
        self.assertTrue(
 
2643
            lines[(b'rancestor\n', self.get_simple_key(b'rancestor'))] > 0)
 
2644
        self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
 
2645
        self.assertTrue(
 
2646
            lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2616
2647
 
2617
2648
    def test_make_mpdiffs(self):
2618
 
        from bzrlib import multiparent
 
2649
        from breezy import multiparent
2619
2650
        files = self.get_versionedfiles('source')
2620
2651
        # add texts that should trip the knit maximum delta chain threshold
2621
2652
        # as well as doing parallel chains of data in knits.
2622
2653
        # this is done by two chains of 25 insertions
2623
 
        files.add_lines(self.get_simple_key('base'), [], ['line\n'])
2624
 
        files.add_lines(self.get_simple_key('noeol'),
2625
 
            self.get_parents([self.get_simple_key('base')]), ['line'])
 
2654
        files.add_lines(self.get_simple_key(b'base'), [], [b'line\n'])
 
2655
        files.add_lines(self.get_simple_key(b'noeol'),
 
2656
                        self.get_parents([self.get_simple_key(b'base')]), [b'line'])
2626
2657
        # detailed eol tests:
2627
2658
        # shared last line with parent no-eol
2628
 
        files.add_lines(self.get_simple_key('noeolsecond'),
2629
 
            self.get_parents([self.get_simple_key('noeol')]),
2630
 
                ['line\n', 'line'])
 
2659
        files.add_lines(self.get_simple_key(b'noeolsecond'),
 
2660
                        self.get_parents([self.get_simple_key(b'noeol')]),
 
2661
                        [b'line\n', b'line'])
2631
2662
        # differing last line with parent, both no-eol
2632
 
        files.add_lines(self.get_simple_key('noeolnotshared'),
2633
 
            self.get_parents([self.get_simple_key('noeolsecond')]),
2634
 
                ['line\n', 'phone'])
 
2663
        files.add_lines(self.get_simple_key(b'noeolnotshared'),
 
2664
                        self.get_parents(
 
2665
                            [self.get_simple_key(b'noeolsecond')]),
 
2666
                        [b'line\n', b'phone'])
2635
2667
        # add eol following a noneol parent, change content
2636
 
        files.add_lines(self.get_simple_key('eol'),
2637
 
            self.get_parents([self.get_simple_key('noeol')]), ['phone\n'])
 
2668
        files.add_lines(self.get_simple_key(b'eol'),
 
2669
                        self.get_parents([self.get_simple_key(b'noeol')]), [b'phone\n'])
2638
2670
        # add eol following a noneol parent, no change content
2639
 
        files.add_lines(self.get_simple_key('eolline'),
2640
 
            self.get_parents([self.get_simple_key('noeol')]), ['line\n'])
 
2671
        files.add_lines(self.get_simple_key(b'eolline'),
 
2672
                        self.get_parents([self.get_simple_key(b'noeol')]), [b'line\n'])
2641
2673
        # noeol with no parents:
2642
 
        files.add_lines(self.get_simple_key('noeolbase'), [], ['line'])
 
2674
        files.add_lines(self.get_simple_key(b'noeolbase'), [], [b'line'])
2643
2675
        # noeol preceeding its leftmost parent in the output:
2644
2676
        # this is done by making it a merge of two parents with no common
2645
2677
        # anestry: noeolbase and noeol with the
2646
2678
        # later-inserted parent the leftmost.
2647
 
        files.add_lines(self.get_simple_key('eolbeforefirstparent'),
2648
 
            self.get_parents([self.get_simple_key('noeolbase'),
2649
 
                self.get_simple_key('noeol')]),
2650
 
            ['line'])
 
2679
        files.add_lines(self.get_simple_key(b'eolbeforefirstparent'),
 
2680
                        self.get_parents([self.get_simple_key(b'noeolbase'),
 
2681
                                          self.get_simple_key(b'noeol')]),
 
2682
                        [b'line'])
2651
2683
        # two identical eol texts
2652
 
        files.add_lines(self.get_simple_key('noeoldup'),
2653
 
            self.get_parents([self.get_simple_key('noeol')]), ['line'])
2654
 
        next_parent = self.get_simple_key('base')
2655
 
        text_name = 'chain1-'
2656
 
        text = ['line\n']
2657
 
        sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2658
 
                 1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2659
 
                 2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2660
 
                 3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2661
 
                 4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
2662
 
                 5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
2663
 
                 6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2664
 
                 7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2665
 
                 8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
2666
 
                 9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2667
 
                 10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2668
 
                 11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2669
 
                 12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
2670
 
                 13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2671
 
                 14:'2c4b1736566b8ca6051e668de68650686a3922f2',
2672
 
                 15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2673
 
                 16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
2674
 
                 17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2675
 
                 18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
2676
 
                 19:'1ebed371807ba5935958ad0884595126e8c4e823',
2677
 
                 20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2678
 
                 21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2679
 
                 22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2680
 
                 23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2681
 
                 24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2682
 
                 25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
 
2684
        files.add_lines(self.get_simple_key(b'noeoldup'),
 
2685
                        self.get_parents([self.get_simple_key(b'noeol')]), [b'line'])
 
2686
        next_parent = self.get_simple_key(b'base')
 
2687
        text_name = b'chain1-'
 
2688
        text = [b'line\n']
 
2689
        sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
 
2690
                 1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
 
2691
                 2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
 
2692
                 3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
 
2693
                 4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
 
2694
                 5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
 
2695
                 6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
 
2696
                 7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
 
2697
                 8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
 
2698
                 9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
 
2699
                 10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
 
2700
                 11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
 
2701
                 12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
 
2702
                 13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
 
2703
                 14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
 
2704
                 15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
 
2705
                 16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
 
2706
                 17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
 
2707
                 18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
 
2708
                 19: b'1ebed371807ba5935958ad0884595126e8c4e823',
 
2709
                 20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
 
2710
                 21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
 
2711
                 22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
 
2712
                 23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
 
2713
                 24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
 
2714
                 25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2683
2715
                 }
2684
2716
        for depth in range(26):
2685
 
            new_version = self.get_simple_key(text_name + '%s' % depth)
2686
 
            text = text + ['line\n']
 
2717
            new_version = self.get_simple_key(text_name + b'%d' % depth)
 
2718
            text = text + [b'line\n']
2687
2719
            files.add_lines(new_version, self.get_parents([next_parent]), text)
2688
2720
            next_parent = new_version
2689
 
        next_parent = self.get_simple_key('base')
2690
 
        text_name = 'chain2-'
2691
 
        text = ['line\n']
 
2721
        next_parent = self.get_simple_key(b'base')
 
2722
        text_name = b'chain2-'
 
2723
        text = [b'line\n']
2692
2724
        for depth in range(26):
2693
 
            new_version = self.get_simple_key(text_name + '%s' % depth)
2694
 
            text = text + ['line\n']
 
2725
            new_version = self.get_simple_key(text_name + b'%d' % depth)
 
2726
            text = text + [b'line\n']
2695
2727
            files.add_lines(new_version, self.get_parents([next_parent]), text)
2696
2728
            next_parent = new_version
2697
2729
        target = self.get_versionedfiles('target')
2701
2733
            target.add_mpdiffs(
2702
2734
                [(key, parents, files.get_sha1s([key])[key], mpdiff)])
2703
2735
            self.assertEqualDiff(
2704
 
                files.get_record_stream([key], 'unordered',
2705
 
                    True).next().get_bytes_as('fulltext'),
2706
 
                target.get_record_stream([key], 'unordered',
2707
 
                    True).next().get_bytes_as('fulltext')
 
2736
                next(files.get_record_stream([key], 'unordered',
 
2737
                                             True)).get_bytes_as('fulltext'),
 
2738
                next(target.get_record_stream([key], 'unordered',
 
2739
                                              True)).get_bytes_as('fulltext')
2708
2740
                )
2709
2741
 
2710
2742
    def test_keys(self):
2713
2745
        files = self.get_versionedfiles()
2714
2746
        self.assertEqual(set(), set(files.keys()))
2715
2747
        if self.key_length == 1:
2716
 
            key = ('foo',)
 
2748
            key = (b'foo',)
2717
2749
        else:
2718
 
            key = ('foo', 'bar',)
 
2750
            key = (b'foo', b'bar',)
2719
2751
        files.add_lines(key, (), [])
2720
 
        self.assertEqual(set([key]), set(files.keys()))
 
2752
        self.assertEqual({key}, set(files.keys()))
2721
2753
 
2722
2754
 
2723
2755
class VirtualVersionedFilesTests(TestCase):
2731
2763
        return ret
2732
2764
 
2733
2765
    def setUp(self):
2734
 
        TestCase.setUp(self)
 
2766
        super(VirtualVersionedFilesTests, self).setUp()
2735
2767
        self._lines = {}
2736
2768
        self._parent_map = {}
2737
2769
        self.texts = VirtualVersionedFiles(self._get_parent_map,
2739
2771
 
2740
2772
    def test_add_lines(self):
2741
2773
        self.assertRaises(NotImplementedError,
2742
 
                self.texts.add_lines, "foo", [], [])
 
2774
                          self.texts.add_lines, b"foo", [], [])
2743
2775
 
2744
2776
    def test_add_mpdiffs(self):
2745
2777
        self.assertRaises(NotImplementedError,
2746
 
                self.texts.add_mpdiffs, [])
 
2778
                          self.texts.add_mpdiffs, [])
2747
2779
 
2748
2780
    def test_check_noerrors(self):
2749
2781
        self.texts.check()
2753
2785
                          [])
2754
2786
 
2755
2787
    def test_get_sha1s_nonexistent(self):
2756
 
        self.assertEquals({}, self.texts.get_sha1s([("NONEXISTENT",)]))
 
2788
        self.assertEqual({}, self.texts.get_sha1s([(b"NONEXISTENT",)]))
2757
2789
 
2758
2790
    def test_get_sha1s(self):
2759
 
        self._lines["key"] = ["dataline1", "dataline2"]
2760
 
        self.assertEquals({("key",): osutils.sha_strings(self._lines["key"])},
2761
 
                           self.texts.get_sha1s([("key",)]))
 
2791
        self._lines[b"key"] = [b"dataline1", b"dataline2"]
 
2792
        self.assertEqual({(b"key",): osutils.sha_strings(self._lines[b"key"])},
 
2793
                         self.texts.get_sha1s([(b"key",)]))
2762
2794
 
2763
2795
    def test_get_parent_map(self):
2764
 
        self._parent_map = {"G": ("A", "B")}
2765
 
        self.assertEquals({("G",): (("A",),("B",))},
2766
 
                          self.texts.get_parent_map([("G",), ("L",)]))
 
2796
        self._parent_map = {b"G": (b"A", b"B")}
 
2797
        self.assertEqual({(b"G",): ((b"A",), (b"B",))},
 
2798
                         self.texts.get_parent_map([(b"G",), (b"L",)]))
2767
2799
 
2768
2800
    def test_get_record_stream(self):
2769
 
        self._lines["A"] = ["FOO", "BAR"]
2770
 
        it = self.texts.get_record_stream([("A",)], "unordered", True)
2771
 
        record = it.next()
2772
 
        self.assertEquals("chunked", record.storage_kind)
2773
 
        self.assertEquals("FOOBAR", record.get_bytes_as("fulltext"))
2774
 
        self.assertEquals(["FOO", "BAR"], record.get_bytes_as("chunked"))
 
2801
        self._lines[b"A"] = [b"FOO", b"BAR"]
 
2802
        it = self.texts.get_record_stream([(b"A",)], "unordered", True)
 
2803
        record = next(it)
 
2804
        self.assertEqual("chunked", record.storage_kind)
 
2805
        self.assertEqual(b"FOOBAR", record.get_bytes_as("fulltext"))
 
2806
        self.assertEqual([b"FOO", b"BAR"], record.get_bytes_as("chunked"))
2775
2807
 
2776
2808
    def test_get_record_stream_absent(self):
2777
 
        it = self.texts.get_record_stream([("A",)], "unordered", True)
2778
 
        record = it.next()
2779
 
        self.assertEquals("absent", record.storage_kind)
 
2809
        it = self.texts.get_record_stream([(b"A",)], "unordered", True)
 
2810
        record = next(it)
 
2811
        self.assertEqual("absent", record.storage_kind)
2780
2812
 
2781
2813
    def test_iter_lines_added_or_present_in_keys(self):
2782
 
        self._lines["A"] = ["FOO", "BAR"]
2783
 
        self._lines["B"] = ["HEY"]
2784
 
        self._lines["C"] = ["Alberta"]
2785
 
        it = self.texts.iter_lines_added_or_present_in_keys([("A",), ("B",)])
2786
 
        self.assertEquals(sorted([("FOO", "A"), ("BAR", "A"), ("HEY", "B")]),
2787
 
            sorted(list(it)))
 
2814
        self._lines[b"A"] = [b"FOO", b"BAR"]
 
2815
        self._lines[b"B"] = [b"HEY"]
 
2816
        self._lines[b"C"] = [b"Alberta"]
 
2817
        it = self.texts.iter_lines_added_or_present_in_keys([(b"A",), (b"B",)])
 
2818
        self.assertEqual(sorted([(b"FOO", b"A"), (b"BAR", b"A"), (b"HEY", b"B")]),
 
2819
                         sorted(list(it)))
2788
2820
 
2789
2821
 
2790
2822
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2792
2824
    def get_ordering_vf(self, key_priority):
2793
2825
        builder = self.make_branch_builder('test')
2794
2826
        builder.start_series()
2795
 
        builder.build_snapshot('A', None, [
2796
 
            ('add', ('', 'TREE_ROOT', 'directory', None))])
2797
 
        builder.build_snapshot('B', ['A'], [])
2798
 
        builder.build_snapshot('C', ['B'], [])
2799
 
        builder.build_snapshot('D', ['C'], [])
 
2827
        builder.build_snapshot(None, [
 
2828
            ('add', ('', b'TREE_ROOT', 'directory', None))],
 
2829
            revision_id=b'A')
 
2830
        builder.build_snapshot([b'A'], [], revision_id=b'B')
 
2831
        builder.build_snapshot([b'B'], [], revision_id=b'C')
 
2832
        builder.build_snapshot([b'C'], [], revision_id=b'D')
2800
2833
        builder.finish_series()
2801
2834
        b = builder.get_branch()
2802
2835
        b.lock_read()
2809
2842
        self.assertEqual([], vf.calls)
2810
2843
 
2811
2844
    def test_get_record_stream_topological(self):
2812
 
        vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2813
 
        request_keys = [('B',), ('C',), ('D',), ('A',)]
 
2845
        vf = self.get_ordering_vf(
 
2846
            {(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
 
2847
        request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2814
2848
        keys = [r.key for r in vf.get_record_stream(request_keys,
2815
 
                                    'topological', False)]
 
2849
                                                    'topological', False)]
2816
2850
        # We should have gotten the keys in topological order
2817
 
        self.assertEqual([('A',), ('B',), ('C',), ('D',)], keys)
 
2851
        self.assertEqual([(b'A',), (b'B',), (b'C',), (b'D',)], keys)
2818
2852
        # And recorded that the request was made
2819
2853
        self.assertEqual([('get_record_stream', request_keys, 'topological',
2820
2854
                           False)], vf.calls)
2821
2855
 
2822
2856
    def test_get_record_stream_ordered(self):
2823
 
        vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2824
 
        request_keys = [('B',), ('C',), ('D',), ('A',)]
 
2857
        vf = self.get_ordering_vf(
 
2858
            {(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
 
2859
        request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2825
2860
        keys = [r.key for r in vf.get_record_stream(request_keys,
2826
 
                                   'unordered', False)]
 
2861
                                                    'unordered', False)]
2827
2862
        # They should be returned based on their priority
2828
 
        self.assertEqual([('D',), ('B',), ('A',), ('C',)], keys)
 
2863
        self.assertEqual([(b'D',), (b'B',), (b'A',), (b'C',)], keys)
2829
2864
        # And the request recorded
2830
2865
        self.assertEqual([('get_record_stream', request_keys, 'unordered',
2831
2866
                           False)], vf.calls)
2832
2867
 
2833
2868
    def test_get_record_stream_implicit_order(self):
2834
 
        vf = self.get_ordering_vf({('B',): 2, ('D',): 1})
2835
 
        request_keys = [('B',), ('C',), ('D',), ('A',)]
 
2869
        vf = self.get_ordering_vf({(b'B',): 2, (b'D',): 1})
 
2870
        request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2836
2871
        keys = [r.key for r in vf.get_record_stream(request_keys,
2837
 
                                   'unordered', False)]
 
2872
                                                    'unordered', False)]
2838
2873
        # A and C are not in the map, so they get sorted to the front. A comes
2839
2874
        # before C alphabetically, so it comes back first
2840
 
        self.assertEqual([('A',), ('C',), ('D',), ('B',)], keys)
 
2875
        self.assertEqual([(b'A',), (b'C',), (b'D',), (b'B',)], keys)
2841
2876
        # And the request recorded
2842
2877
        self.assertEqual([('get_record_stream', request_keys, 'unordered',
2843
2878
                           False)], vf.calls)