1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
24
from gzip import GzipFile
39
from ..errors import (
41
RevisionAlreadyPresent,
43
from ..bzr.knit import (
48
from ..sixish import (
54
TestCaseWithMemoryTransport,
58
from .http_utils import TestCaseWithWebserver
59
from ..transport.memory import MemoryTransport
60
from ..bzr import versionedfile as versionedfile
61
from ..bzr.versionedfile import (
63
HashEscapedPrefixMapper,
65
VirtualVersionedFiles,
66
make_versioned_files_factory,
68
from ..bzr.weave import (
72
from ..bzr.weavefile import write_weave
73
from .scenarios import load_tests_apply_scenarios
76
load_tests = load_tests_apply_scenarios
79
def get_diamond_vf(f, trailing_eol=True, left_only=False):
80
"""Get a diamond graph to exercise deltas and merges.
82
:param trailing_eol: If True end the last line with \n.
86
b'base': ((b'origin',),),
87
b'left': ((b'base',),),
88
b'right': ((b'base',),),
89
b'merged': ((b'left',), (b'right',)),
91
# insert a diamond graph to exercise deltas and merges.
96
f.add_lines(b'origin', [], [b'origin' + last_char])
97
f.add_lines(b'base', [b'origin'], [b'base' + last_char])
98
f.add_lines(b'left', [b'base'], [b'base\n', b'left' + last_char])
100
f.add_lines(b'right', [b'base'],
101
[b'base\n', b'right' + last_char])
102
f.add_lines(b'merged', [b'left', b'right'],
103
[b'base\n', b'left\n', b'right\n', b'merged' + last_char])
107
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
108
nograph=False, nokeys=False):
109
"""Get a diamond graph to exercise deltas and merges.
111
This creates a 5-node graph in files. If files supports 2-length keys two
112
graphs are made to exercise the support for multiple ids.
114
:param trailing_eol: If True end the last line with \n.
115
:param key_length: The length of keys in files. Currently supports length 1
117
:param left_only: If True do not add the right and merged nodes.
118
:param nograph: If True, do not provide parents to the add_lines calls;
119
this is useful for tests that need inserted data but have graphless
121
:param nokeys: If True, pass None is as the key for all insertions.
122
Currently implies nograph.
123
:return: The results of the add_lines calls.
130
prefixes = [(b'FileA',), (b'FileB',)]
131
# insert a diamond graph to exercise deltas and merges.
137
def get_parents(suffix_list):
141
result = [prefix + suffix for suffix in suffix_list]
148
# we loop over each key because that spreads the inserts across prefixes,
149
# which is how commit operates.
150
for prefix in prefixes:
151
result.append(files.add_lines(prefix + get_key(b'origin'), (),
152
[b'origin' + last_char]))
153
for prefix in prefixes:
154
result.append(files.add_lines(prefix + get_key(b'base'),
155
get_parents([(b'origin',)]), [b'base' + last_char]))
156
for prefix in prefixes:
157
result.append(files.add_lines(prefix + get_key(b'left'),
158
get_parents([(b'base',)]),
159
[b'base\n', b'left' + last_char]))
161
for prefix in prefixes:
162
result.append(files.add_lines(prefix + get_key(b'right'),
163
get_parents([(b'base',)]),
164
[b'base\n', b'right' + last_char]))
165
for prefix in prefixes:
166
result.append(files.add_lines(prefix + get_key(b'merged'),
167
get_parents([(b'left',), (b'right',)]),
168
[b'base\n', b'left\n', b'right\n', b'merged' + last_char]))
172
class VersionedFileTestMixIn(object):
173
"""A mixin test class for testing VersionedFiles.
175
This is not an adaptor-style test at this point because
176
theres no dynamic substitution of versioned file implementations,
177
they are strictly controlled by their owning repositories.
180
def get_transaction(self):
181
if not hasattr(self, '_transaction'):
182
self._transaction = None
183
return self._transaction
187
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
188
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
190
versions = f.versions()
191
self.assertTrue(b'r0' in versions)
192
self.assertTrue(b'r1' in versions)
193
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
194
self.assertEqual(f.get_text(b'r0'), b'a\nb\n')
195
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
196
self.assertEqual(2, len(f))
197
self.assertEqual(2, f.num_versions())
199
self.assertRaises(RevisionNotPresent,
200
f.add_lines, b'r2', [b'foo'], [])
201
self.assertRaises(RevisionAlreadyPresent,
202
f.add_lines, b'r1', [], [])
204
# this checks that reopen with create=True does not break anything.
205
f = self.reopen_file(create=True)
208
def test_adds_with_parent_texts(self):
211
_, _, parent_texts[b'r0'] = f.add_lines(b'r0', [], [b'a\n', b'b\n'])
213
_, _, parent_texts[b'r1'] = f.add_lines_with_ghosts(b'r1',
214
[b'r0', b'ghost'], [b'b\n', b'c\n'], parent_texts=parent_texts)
215
except NotImplementedError:
216
# if the format doesn't support ghosts, just add normally.
217
_, _, parent_texts[b'r1'] = f.add_lines(b'r1',
218
[b'r0'], [b'b\n', b'c\n'], parent_texts=parent_texts)
219
f.add_lines(b'r2', [b'r1'], [b'c\n', b'd\n'], parent_texts=parent_texts)
220
self.assertNotEqual(None, parent_texts[b'r0'])
221
self.assertNotEqual(None, parent_texts[b'r1'])
223
versions = f.versions()
224
self.assertTrue(b'r0' in versions)
225
self.assertTrue(b'r1' in versions)
226
self.assertTrue(b'r2' in versions)
227
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
228
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
229
self.assertEqual(f.get_lines(b'r2'), [b'c\n', b'd\n'])
230
self.assertEqual(3, f.num_versions())
231
origins = f.annotate(b'r1')
232
self.assertEqual(origins[0][0], b'r0')
233
self.assertEqual(origins[1][0], b'r1')
234
origins = f.annotate(b'r2')
235
self.assertEqual(origins[0][0], b'r1')
236
self.assertEqual(origins[1][0], b'r2')
239
f = self.reopen_file()
242
def test_add_unicode_content(self):
243
# unicode content is not permitted in versioned files.
244
# versioned files version sequences of bytes only.
246
self.assertRaises(errors.BzrBadParameterUnicode,
247
vf.add_lines, b'a', [], [b'a\n', u'b\n', b'c\n'])
249
(errors.BzrBadParameterUnicode, NotImplementedError),
250
vf.add_lines_with_ghosts, b'a', [], [b'a\n', u'b\n', b'c\n'])
252
def test_add_follows_left_matching_blocks(self):
253
"""If we change left_matching_blocks, delta changes
255
Note: There are multiple correct deltas in this case, because
256
we start with 1 "a" and we get 3.
259
if isinstance(vf, WeaveFile):
260
raise TestSkipped("WeaveFile ignores left_matching_blocks")
261
vf.add_lines(b'1', [], [b'a\n'])
262
vf.add_lines(b'2', [b'1'], [b'a\n', b'a\n', b'a\n'],
263
left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
264
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'2'))
265
vf.add_lines(b'3', [b'1'], [b'a\n', b'a\n', b'a\n'],
266
left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
267
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'3'))
269
def test_inline_newline_throws(self):
270
# \r characters are not permitted in lines being added
272
self.assertRaises(errors.BzrBadParameterContainsNewline,
273
vf.add_lines, b'a', [], [b'a\n\n'])
275
(errors.BzrBadParameterContainsNewline, NotImplementedError),
276
vf.add_lines_with_ghosts, b'a', [], [b'a\n\n'])
277
# but inline CR's are allowed
278
vf.add_lines(b'a', [], [b'a\r\n'])
280
vf.add_lines_with_ghosts(b'b', [], [b'a\r\n'])
281
except NotImplementedError:
284
def test_add_reserved(self):
286
self.assertRaises(errors.ReservedId,
287
vf.add_lines, b'a:', [], [b'a\n', b'b\n', b'c\n'])
289
def test_add_lines_nostoresha(self):
290
"""When nostore_sha is supplied using old content raises."""
292
empty_text = (b'a', [])
293
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
294
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
296
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
297
sha, _, _ = vf.add_lines(version, [], lines)
299
# we now have a copy of all the lines in the vf.
300
for sha, (version, lines) in zip(
301
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
302
self.assertRaises(errors.ExistingContent,
303
vf.add_lines, version + b"2", [], lines,
305
# and no new version should have been added.
306
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
309
def test_add_lines_with_ghosts_nostoresha(self):
310
"""When nostore_sha is supplied using old content raises."""
312
empty_text = (b'a', [])
313
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
314
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
316
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
317
sha, _, _ = vf.add_lines(version, [], lines)
319
# we now have a copy of all the lines in the vf.
320
# is the test applicable to this vf implementation?
322
vf.add_lines_with_ghosts(b'd', [], [])
323
except NotImplementedError:
324
raise TestSkipped("add_lines_with_ghosts is optional")
325
for sha, (version, lines) in zip(
326
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
327
self.assertRaises(errors.ExistingContent,
328
vf.add_lines_with_ghosts, version + b"2", [], lines,
330
# and no new version should have been added.
331
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
334
def test_add_lines_return_value(self):
335
# add_lines should return the sha1 and the text size.
337
empty_text = (b'a', [])
338
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
339
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
340
# check results for the three cases:
341
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
342
# the first two elements are the same for all versioned files:
343
# - the digest and the size of the text. For some versioned files
344
# additional data is returned in additional tuple elements.
345
result = vf.add_lines(version, [], lines)
346
self.assertEqual(3, len(result))
347
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
349
# parents should not affect the result:
350
lines = sample_text_nl[1]
351
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
352
vf.add_lines(b'd', [b'b', b'c'], lines)[0:2])
354
def test_get_reserved(self):
356
self.assertRaises(errors.ReservedId, vf.get_texts, [b'b:'])
357
self.assertRaises(errors.ReservedId, vf.get_lines, b'b:')
358
self.assertRaises(errors.ReservedId, vf.get_text, b'b:')
360
def test_add_unchanged_last_line_noeol_snapshot(self):
361
"""Add a text with an unchanged last line with no eol should work."""
362
# Test adding this in a number of chain lengths; because the interface
363
# for VersionedFile does not allow forcing a specific chain length, we
364
# just use a small base to get the first snapshot, then a much longer
365
# first line for the next add (which will make the third add snapshot)
366
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
367
# as a capped delta length, but ideally we would have some way of
368
# tuning the test to the store (e.g. keep going until a snapshot
370
for length in range(20):
372
vf = self.get_file('case-%d' % length)
375
for step in range(length):
376
version = prefix % step
377
lines = ([b'prelude \n'] * step) + [b'line']
378
vf.add_lines(version, parents, lines)
379
version_lines[version] = lines
381
vf.add_lines(b'no-eol', parents, [b'line'])
382
vf.get_texts(version_lines.keys())
383
self.assertEqualDiff(b'line', vf.get_text(b'no-eol'))
385
def test_get_texts_eol_variation(self):
386
# similar to the failure in <http://bugs.launchpad.net/234748>
388
sample_text_nl = [b"line\n"]
389
sample_text_no_nl = [b"line"]
396
lines = sample_text_nl
398
lines = sample_text_no_nl
399
# left_matching blocks is an internal api; it operates on the
400
# *internal* representation for a knit, which is with *all* lines
401
# being normalised to end with \n - even the final line in a no_nl
402
# file. Using it here ensures that a broken internal implementation
403
# (which is what this test tests) will generate a correct line
404
# delta (which is to say, an empty delta).
405
vf.add_lines(version, parents, lines,
406
left_matching_blocks=[(0, 0, 1)])
408
versions.append(version)
409
version_lines[version] = lines
411
vf.get_texts(versions)
412
vf.get_texts(reversed(versions))
414
def test_add_lines_with_matching_blocks_noeol_last_line(self):
415
"""Add a text with an unchanged last line with no eol should work."""
416
from breezy import multiparent
417
# Hand verified sha1 of the text we're adding.
418
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
419
# Create a mpdiff which adds a new line before the trailing line, and
420
# reuse the last line unaltered (which can cause annotation reuse).
421
# Test adding this in two situations:
422
# On top of a new insertion
423
vf = self.get_file('fulltext')
424
vf.add_lines(b'noeol', [], [b'line'])
425
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
426
left_matching_blocks=[(0, 1, 1)])
427
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
429
vf = self.get_file('delta')
430
vf.add_lines(b'base', [], [b'line'])
431
vf.add_lines(b'noeol', [b'base'], [b'prelude\n', b'line'])
432
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
433
left_matching_blocks=[(1, 1, 1)])
434
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
436
def test_make_mpdiffs(self):
437
from breezy import multiparent
438
vf = self.get_file('foo')
439
sha1s = self._setup_for_deltas(vf)
440
new_vf = self.get_file('bar')
441
for version in multiparent.topo_iter(vf):
442
mpdiff = vf.make_mpdiffs([version])[0]
443
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
444
vf.get_sha1s([version])[version], mpdiff)])
445
self.assertEqualDiff(vf.get_text(version),
446
new_vf.get_text(version))
448
def test_make_mpdiffs_with_ghosts(self):
449
vf = self.get_file('foo')
451
vf.add_lines_with_ghosts(b'text', [b'ghost'], [b'line\n'])
452
except NotImplementedError:
453
# old Weave formats do not allow ghosts
455
self.assertRaises(errors.RevisionNotPresent, vf.make_mpdiffs, [b'ghost'])
457
def _setup_for_deltas(self, f):
458
self.assertFalse(f.has_version('base'))
459
# add texts that should trip the knit maximum delta chain threshold
460
# as well as doing parallel chains of data in knits.
461
# this is done by two chains of 25 insertions
462
f.add_lines(b'base', [], [b'line\n'])
463
f.add_lines(b'noeol', [b'base'], [b'line'])
464
# detailed eol tests:
465
# shared last line with parent no-eol
466
f.add_lines(b'noeolsecond', [b'noeol'], [b'line\n', b'line'])
467
# differing last line with parent, both no-eol
468
f.add_lines(b'noeolnotshared', [b'noeolsecond'], [b'line\n', b'phone'])
469
# add eol following a noneol parent, change content
470
f.add_lines(b'eol', [b'noeol'], [b'phone\n'])
471
# add eol following a noneol parent, no change content
472
f.add_lines(b'eolline', [b'noeol'], [b'line\n'])
473
# noeol with no parents:
474
f.add_lines(b'noeolbase', [], [b'line'])
475
# noeol preceeding its leftmost parent in the output:
476
# this is done by making it a merge of two parents with no common
477
# anestry: noeolbase and noeol with the
478
# later-inserted parent the leftmost.
479
f.add_lines(b'eolbeforefirstparent', [b'noeolbase', b'noeol'], [b'line'])
480
# two identical eol texts
481
f.add_lines(b'noeoldup', [b'noeol'], [b'line'])
482
next_parent = b'base'
483
text_name = b'chain1-'
485
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
486
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
487
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
488
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
489
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
490
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
491
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
492
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
493
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
494
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
495
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
496
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
497
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
498
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
499
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
500
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
501
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
502
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
503
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
504
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
505
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
506
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
507
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
508
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
509
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
510
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
512
for depth in range(26):
513
new_version = text_name + b'%d' % depth
514
text = text + [b'line\n']
515
f.add_lines(new_version, [next_parent], text)
516
next_parent = new_version
517
next_parent = b'base'
518
text_name = b'chain2-'
520
for depth in range(26):
521
new_version = text_name + b'%d' % depth
522
text = text + [b'line\n']
523
f.add_lines(new_version, [next_parent], text)
524
next_parent = new_version
527
def test_ancestry(self):
529
self.assertEqual([], f.get_ancestry([]))
530
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
531
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
532
f.add_lines(b'r2', [b'r0'], [b'b\n', b'c\n'])
533
f.add_lines(b'r3', [b'r2'], [b'b\n', b'c\n'])
534
f.add_lines(b'rM', [b'r1', b'r2'], [b'b\n', b'c\n'])
535
self.assertEqual([], f.get_ancestry([]))
536
versions = f.get_ancestry([b'rM'])
537
# there are some possibilities:
541
# so we check indexes
542
r0 = versions.index(b'r0')
543
r1 = versions.index(b'r1')
544
r2 = versions.index(b'r2')
545
self.assertFalse(b'r3' in versions)
546
rM = versions.index(b'rM')
547
self.assertTrue(r0 < r1)
548
self.assertTrue(r0 < r2)
549
self.assertTrue(r1 < rM)
550
self.assertTrue(r2 < rM)
552
self.assertRaises(RevisionNotPresent,
553
f.get_ancestry, [b'rM', b'rX'])
555
self.assertEqual(set(f.get_ancestry(b'rM')),
556
set(f.get_ancestry(b'rM', topo_sorted=False)))
558
def test_mutate_after_finish(self):
559
self._transaction = 'before'
561
self._transaction = 'after'
562
self.assertRaises(errors.OutSideTransaction, f.add_lines, b'', [], [])
563
self.assertRaises(errors.OutSideTransaction, f.add_lines_with_ghosts, b'', [], [])
565
def test_copy_to(self):
567
f.add_lines(b'0', [], [b'a\n'])
568
t = MemoryTransport()
570
for suffix in self.get_factory().get_suffixes():
571
self.assertTrue(t.has('foo' + suffix))
573
def test_get_suffixes(self):
575
# and should be a list
576
self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
578
def test_get_parent_map(self):
580
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
582
{b'r0':()}, f.get_parent_map([b'r0']))
583
f.add_lines(b'r1', [b'r0'], [b'a\n', b'b\n'])
585
{b'r1':(b'r0',)}, f.get_parent_map([b'r1']))
589
f.get_parent_map([b'r0', b'r1']))
590
f.add_lines(b'r2', [], [b'a\n', b'b\n'])
591
f.add_lines(b'r3', [], [b'a\n', b'b\n'])
592
f.add_lines(b'm', [b'r0', b'r1', b'r2', b'r3'], [b'a\n', b'b\n'])
594
{b'm':(b'r0', b'r1', b'r2', b'r3')}, f.get_parent_map([b'm']))
595
self.assertEqual({}, f.get_parent_map(b'y'))
599
f.get_parent_map([b'r0', b'y', b'r1']))
601
def test_annotate(self):
603
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
604
f.add_lines(b'r1', [b'r0'], [b'c\n', b'b\n'])
605
origins = f.annotate(b'r1')
606
self.assertEqual(origins[0][0], b'r1')
607
self.assertEqual(origins[1][0], b'r0')
609
self.assertRaises(RevisionNotPresent,
612
def test_detection(self):
613
# Test weaves detect corruption.
615
# Weaves contain a checksum of their texts.
616
# When a text is extracted, this checksum should be
619
w = self.get_file_corrupted_text()
621
self.assertEqual(b'hello\n', w.get_text(b'v1'))
622
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
623
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
624
self.assertRaises(WeaveInvalidChecksum, w.check)
626
w = self.get_file_corrupted_checksum()
628
self.assertEqual(b'hello\n', w.get_text(b'v1'))
629
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
630
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
631
self.assertRaises(WeaveInvalidChecksum, w.check)
633
def get_file_corrupted_text(self):
634
"""Return a versioned file with corrupt text but valid metadata."""
635
raise NotImplementedError(self.get_file_corrupted_text)
637
def reopen_file(self, name='foo'):
638
"""Open the versioned file from disk again."""
639
raise NotImplementedError(self.reopen_file)
641
def test_iter_lines_added_or_present_in_versions(self):
642
# test that we get at least an equalset of the lines added by
643
# versions in the weave
644
# the ordering here is to make a tree so that dumb searches have
645
# more changes to muck up.
647
class InstrumentedProgress(progress.ProgressTask):
650
progress.ProgressTask.__init__(self)
653
def update(self, msg=None, current=None, total=None):
654
self.updates.append((msg, current, total))
657
# add a base to get included
658
vf.add_lines(b'base', [], [b'base\n'])
659
# add a ancestor to be included on one side
660
vf.add_lines(b'lancestor', [], [b'lancestor\n'])
661
# add a ancestor to be included on the other side
662
vf.add_lines(b'rancestor', [b'base'], [b'rancestor\n'])
663
# add a child of rancestor with no eofile-nl
664
vf.add_lines(b'child', [b'rancestor'], [b'base\n', b'child\n'])
665
# add a child of lancestor and base to join the two roots
666
vf.add_lines(b'otherchild',
667
[b'lancestor', b'base'],
668
[b'base\n', b'lancestor\n', b'otherchild\n'])
669
def iter_with_versions(versions, expected):
670
# now we need to see what lines are returned, and how often.
672
progress = InstrumentedProgress()
673
# iterate over the lines
674
for line in vf.iter_lines_added_or_present_in_versions(versions,
676
lines.setdefault(line, 0)
678
if []!= progress.updates:
679
self.assertEqual(expected, progress.updates)
681
lines = iter_with_versions([b'child', b'otherchild'],
682
[('Walking content', 0, 2),
683
('Walking content', 1, 2),
684
('Walking content', 2, 2)])
685
# we must see child and otherchild
686
self.assertTrue(lines[(b'child\n', b'child')] > 0)
687
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
688
# we dont care if we got more than that.
691
lines = iter_with_versions(None, [('Walking content', 0, 5),
692
('Walking content', 1, 5),
693
('Walking content', 2, 5),
694
('Walking content', 3, 5),
695
('Walking content', 4, 5),
696
('Walking content', 5, 5)])
697
# all lines must be seen at least once
698
self.assertTrue(lines[(b'base\n', b'base')] > 0)
699
self.assertTrue(lines[(b'lancestor\n', b'lancestor')] > 0)
700
self.assertTrue(lines[(b'rancestor\n', b'rancestor')] > 0)
701
self.assertTrue(lines[(b'child\n', b'child')] > 0)
702
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
704
def test_add_lines_with_ghosts(self):
705
# some versioned file formats allow lines to be added with parent
706
# information that is > than that in the format. Formats that do
707
# not support this need to raise NotImplementedError on the
708
# add_lines_with_ghosts api.
710
# add a revision with ghost parents
711
# The preferred form is utf8, but we should translate when needed
712
parent_id_unicode = u'b\xbfse'
713
parent_id_utf8 = parent_id_unicode.encode('utf8')
715
vf.add_lines_with_ghosts(b'notbxbfse', [parent_id_utf8], [])
716
except NotImplementedError:
717
# check the other ghost apis are also not implemented
718
self.assertRaises(NotImplementedError, vf.get_ancestry_with_ghosts, [b'foo'])
719
self.assertRaises(NotImplementedError, vf.get_parents_with_ghosts, b'foo')
721
vf = self.reopen_file()
722
# test key graph related apis: getncestry, _graph, get_parents
724
# - these are ghost unaware and must not be reflect ghosts
725
self.assertEqual([b'notbxbfse'], vf.get_ancestry(b'notbxbfse'))
726
self.assertFalse(vf.has_version(parent_id_utf8))
727
# we have _with_ghost apis to give us ghost information.
728
self.assertEqual([parent_id_utf8, b'notbxbfse'], vf.get_ancestry_with_ghosts([b'notbxbfse']))
729
self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts(b'notbxbfse'))
730
# if we add something that is a ghost of another, it should correct the
731
# results of the prior apis
732
vf.add_lines(parent_id_utf8, [], [])
733
self.assertEqual([parent_id_utf8, b'notbxbfse'], vf.get_ancestry([b'notbxbfse']))
734
self.assertEqual({b'notbxbfse':(parent_id_utf8,)},
735
vf.get_parent_map([b'notbxbfse']))
736
self.assertTrue(vf.has_version(parent_id_utf8))
737
# we have _with_ghost apis to give us ghost information.
738
self.assertEqual([parent_id_utf8, b'notbxbfse'],
739
vf.get_ancestry_with_ghosts([b'notbxbfse']))
740
self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts(b'notbxbfse'))
742
def test_add_lines_with_ghosts_after_normal_revs(self):
743
# some versioned file formats allow lines to be added with parent
744
# information that is > than that in the format. Formats that do
745
# not support this need to raise NotImplementedError on the
746
# add_lines_with_ghosts api.
748
# probe for ghost support
750
vf.add_lines_with_ghosts(b'base', [], [b'line\n', b'line_b\n'])
751
except NotImplementedError:
753
vf.add_lines_with_ghosts(b'references_ghost',
754
[b'base', b'a_ghost'],
755
[b'line\n', b'line_b\n', b'line_c\n'])
756
origins = vf.annotate(b'references_ghost')
757
self.assertEqual((b'base', b'line\n'), origins[0])
758
self.assertEqual((b'base', b'line_b\n'), origins[1])
759
self.assertEqual((b'references_ghost', b'line_c\n'), origins[2])
761
def test_readonly_mode(self):
762
t = self.get_transport()
763
factory = self.get_factory()
764
vf = factory('id', t, 0o777, create=True, access_mode='w')
765
vf = factory('id', t, access_mode='r')
766
self.assertRaises(errors.ReadOnlyError, vf.add_lines, b'base', [], [])
767
self.assertRaises(errors.ReadOnlyError,
768
vf.add_lines_with_ghosts,
773
def test_get_sha1s(self):
774
# check the sha1 data is available
777
vf.add_lines(b'a', [], [b'a\n'])
778
# the same file, different metadata
779
vf.add_lines(b'b', [b'a'], [b'a\n'])
780
# a file differing only in last newline.
781
vf.add_lines(b'c', [], [b'a'])
783
b'a': b'3f786850e387550fdab836ed7e6dc881de23001b',
784
b'c': b'86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
785
b'b': b'3f786850e387550fdab836ed7e6dc881de23001b',
787
vf.get_sha1s([b'a', b'c', b'b']))
790
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
792
def get_file(self, name='foo'):
793
return WeaveFile(name, self.get_transport(),
795
get_scope=self.get_transaction)
797
def get_file_corrupted_text(self):
798
w = WeaveFile('foo', self.get_transport(),
800
get_scope=self.get_transaction)
801
w.add_lines(b'v1', [], [b'hello\n'])
802
w.add_lines(b'v2', [b'v1'], [b'hello\n', b'there\n'])
804
# We are going to invasively corrupt the text
805
# Make sure the internals of weave are the same
806
self.assertEqual([(b'{', 0)
814
self.assertEqual([b'f572d396fae9206628714fb2ce00f72e94f2258f'
815
, b'90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
820
w._weave[4] = b'There\n'
823
def get_file_corrupted_checksum(self):
824
w = self.get_file_corrupted_text()
826
w._weave[4] = b'there\n'
827
self.assertEqual(b'hello\nthere\n', w.get_text(b'v2'))
829
#Invalid checksum, first digit changed
830
w._sha1s[1] =b'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
833
def reopen_file(self, name='foo', create=False):
834
return WeaveFile(name, self.get_transport(),
836
get_scope=self.get_transaction)
838
def test_no_implicit_create(self):
839
self.assertRaises(errors.NoSuchFile,
842
self.get_transport(),
843
get_scope=self.get_transaction)
845
def get_factory(self):
849
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
852
super(TestPlanMergeVersionedFile, self).setUp()
853
mapper = PrefixMapper()
854
factory = make_file_factory(True, mapper)
855
self.vf1 = factory(self.get_transport('root-1'))
856
self.vf2 = factory(self.get_transport('root-2'))
857
self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
858
self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
860
def test_add_lines(self):
861
self.plan_merge_vf.add_lines((b'root', b'a:'), [], [])
862
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
863
(b'root', b'a'), [], [])
864
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
865
(b'root', b'a:'), None, [])
866
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
867
(b'root', b'a:'), [], None)
869
def setup_abcde(self):
870
self.vf1.add_lines((b'root', b'A'), [], [b'a'])
871
self.vf1.add_lines((b'root', b'B'), [(b'root', b'A')], [b'b'])
872
self.vf2.add_lines((b'root', b'C'), [], [b'c'])
873
self.vf2.add_lines((b'root', b'D'), [(b'root', b'C')], [b'd'])
874
self.plan_merge_vf.add_lines((b'root', b'E:'),
875
[(b'root', b'B'), (b'root', b'D')], [b'e'])
877
def test_get_parents(self):
879
self.assertEqual({(b'root', b'B'): ((b'root', b'A'),)},
880
self.plan_merge_vf.get_parent_map([(b'root', b'B')]))
881
self.assertEqual({(b'root', b'D'): ((b'root', b'C'),)},
882
self.plan_merge_vf.get_parent_map([(b'root', b'D')]))
883
self.assertEqual({(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D'))},
884
self.plan_merge_vf.get_parent_map([(b'root', b'E:')]))
886
self.plan_merge_vf.get_parent_map([(b'root', b'F')]))
888
(b'root', b'B'): ((b'root', b'A'),),
889
(b'root', b'D'): ((b'root', b'C'),),
890
(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D')),
892
self.plan_merge_vf.get_parent_map(
893
[(b'root', b'B'), (b'root', b'D'), (b'root', b'E:'), (b'root', b'F')]))
895
def test_get_record_stream(self):
897
def get_record(suffix):
898
return next(self.plan_merge_vf.get_record_stream(
899
[(b'root', suffix)], 'unordered', True))
900
self.assertEqual(b'a', get_record(b'A').get_bytes_as('fulltext'))
901
self.assertEqual(b'c', get_record(b'C').get_bytes_as('fulltext'))
902
self.assertEqual(b'e', get_record(b'E:').get_bytes_as('fulltext'))
903
self.assertEqual('absent', get_record('F').storage_kind)
906
class TestReadonlyHttpMixin(object):
908
def get_transaction(self):
911
def test_readonly_http_works(self):
912
# we should be able to read from http with a versioned file.
914
# try an empty file access
915
readonly_vf = self.get_factory()('foo',
916
transport.get_transport_from_url(self.get_readonly_url('.')))
917
self.assertEqual([], readonly_vf.versions())
919
def test_readonly_http_works_with_feeling(self):
920
# we should be able to read from http with a versioned file.
923
vf.add_lines(b'1', [], [b'a\n'])
924
vf.add_lines(b'2', [b'1'], [b'b\n', b'a\n'])
925
readonly_vf = self.get_factory()('foo',
926
transport.get_transport_from_url(self.get_readonly_url('.')))
927
self.assertEqual([b'1', b'2'], vf.versions())
928
self.assertEqual([b'1', b'2'], readonly_vf.versions())
929
for version in readonly_vf.versions():
930
readonly_vf.get_lines(version)
933
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
936
return WeaveFile('foo', self.get_transport(),
938
get_scope=self.get_transaction)
940
def get_factory(self):
944
class MergeCasesMixin(object):
946
def doMerge(self, base, a, b, mp):
947
from textwrap import dedent
953
w.add_lines(b'text0', [], list(map(addcrlf, base)))
954
w.add_lines(b'text1', [b'text0'], list(map(addcrlf, a)))
955
w.add_lines(b'text2', [b'text0'], list(map(addcrlf, b)))
959
self.log('merge plan:')
960
p = list(w.plan_merge(b'text1', b'text2'))
961
for state, line in p:
963
self.log('%12s | %s' % (state, line[:-1]))
967
mt.writelines(w.weave_merge(p))
969
self.log(mt.getvalue())
971
mp = list(map(addcrlf, mp))
972
self.assertEqual(mt.readlines(), mp)
974
def testOneInsert(self):
980
def testSeparateInserts(self):
981
self.doMerge([b'aaa', b'bbb', b'ccc'],
982
[b'aaa', b'xxx', b'bbb', b'ccc'],
983
[b'aaa', b'bbb', b'yyy', b'ccc'],
984
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
986
def testSameInsert(self):
987
self.doMerge([b'aaa', b'bbb', b'ccc'],
988
[b'aaa', b'xxx', b'bbb', b'ccc'],
989
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'],
990
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
991
overlappedInsertExpected = [b'aaa', b'xxx', b'yyy', b'bbb']
992
def testOverlappedInsert(self):
993
self.doMerge([b'aaa', b'bbb'],
994
[b'aaa', b'xxx', b'yyy', b'bbb'],
995
[b'aaa', b'xxx', b'bbb'], self.overlappedInsertExpected)
997
# really it ought to reduce this to
998
# [b'aaa', b'xxx', b'yyy', b'bbb']
1001
def testClashReplace(self):
1002
self.doMerge([b'aaa'],
1005
[b'<<<<<<< ', b'xxx', b'=======', b'yyy', b'zzz',
1008
def testNonClashInsert1(self):
1009
self.doMerge([b'aaa'],
1012
[b'<<<<<<< ', b'xxx', b'aaa', b'=======', b'yyy', b'zzz',
1015
def testNonClashInsert2(self):
1016
self.doMerge([b'aaa'],
1021
def testDeleteAndModify(self):
1022
"""Clashing delete and modification.
1024
If one side modifies a region and the other deletes it then
1025
there should be a conflict with one side blank.
1028
#######################################
1029
# skippd, not working yet
1032
self.doMerge([b'aaa', b'bbb', b'ccc'],
1033
[b'aaa', b'ddd', b'ccc'],
1035
[b'<<<<<<<< ', b'aaa', b'=======', b'>>>>>>> ', b'ccc'])
1037
def _test_merge_from_strings(self, base, a, b, expected):
1039
w.add_lines(b'text0', [], base.splitlines(True))
1040
w.add_lines(b'text1', [b'text0'], a.splitlines(True))
1041
w.add_lines(b'text2', [b'text0'], b.splitlines(True))
1042
self.log('merge plan:')
1043
p = list(w.plan_merge(b'text1', b'text2'))
1044
for state, line in p:
1046
self.log('%12s | %s' % (state, line[:-1]))
1047
self.log('merge result:')
1048
result_text = b''.join(w.weave_merge(p))
1049
self.log(result_text)
1050
self.assertEqualDiff(result_text, expected)
1052
def test_weave_merge_conflicts(self):
1053
# does weave merge properly handle plans that end with unchanged?
1054
result = b''.join(self.get_file().weave_merge([('new-a', b'hello\n')]))
1055
self.assertEqual(result, b'hello\n')
1057
def test_deletion_extended(self):
1058
"""One side deletes, the other deletes more.
1079
self._test_merge_from_strings(base, a, b, result)
1081
def test_deletion_overlap(self):
1082
"""Delete overlapping regions with no other conflict.
1084
Arguably it'd be better to treat these as agreement, rather than
1085
conflict, but for now conflict is safer.
1113
self._test_merge_from_strings(base, a, b, result)
1115
def test_agreement_deletion(self):
1116
"""Agree to delete some lines, without conflicts."""
1138
self._test_merge_from_strings(base, a, b, result)
1140
def test_sync_on_deletion(self):
1141
"""Specific case of merge where we can synchronize incorrectly.
1143
A previous version of the weave merge concluded that the two versions
1144
agreed on deleting line 2, and this could be a synchronization point.
1145
Line 1 was then considered in isolation, and thought to be deleted on
1148
It's better to consider the whole thing as a disagreement region.
1159
a's replacement line 2
1172
a's replacement line 2
1179
self._test_merge_from_strings(base, a, b, result)
1182
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1184
def get_file(self, name='foo'):
1185
return WeaveFile(name, self.get_transport(),
1188
def log_contents(self, w):
1189
self.log('weave is:')
1191
write_weave(w, tmpf)
1192
self.log(tmpf.getvalue())
1194
overlappedInsertExpected = [b'aaa', b'<<<<<<< ', b'xxx', b'yyy', b'=======',
1195
b'xxx', b'>>>>>>> ', b'bbb']
1198
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1200
def test_select_adaptor(self):
1201
"""Test expected adapters exist."""
1202
# One scenario for each lookup combination we expect to use.
1203
# Each is source_kind, requested_kind, adapter class
1205
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1206
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1207
('knit-annotated-delta-gz', 'knit-delta-gz',
1208
_mod_knit.DeltaAnnotatedToUnannotated),
1209
('knit-annotated-delta-gz', 'fulltext',
1210
_mod_knit.DeltaAnnotatedToFullText),
1211
('knit-annotated-ft-gz', 'knit-ft-gz',
1212
_mod_knit.FTAnnotatedToUnannotated),
1213
('knit-annotated-ft-gz', 'fulltext',
1214
_mod_knit.FTAnnotatedToFullText),
1216
for source, requested, klass in scenarios:
1217
adapter_factory = versionedfile.adapter_registry.get(
1218
(source, requested))
1219
adapter = adapter_factory(None)
1220
self.assertIsInstance(adapter, klass)
1222
def get_knit(self, annotated=True):
1223
mapper = ConstantMapper('knit')
1224
transport = self.get_transport()
1225
return make_file_factory(annotated, mapper)(transport)
1227
def helpGetBytes(self, f, ft_adapter, delta_adapter):
1228
"""Grab the interested adapted texts for tests."""
1229
# origin is a fulltext
1230
entries = f.get_record_stream([(b'origin',)], 'unordered', False)
1231
base = next(entries)
1232
ft_data = ft_adapter.get_bytes(base)
1233
# merged is both a delta and multiple parents.
1234
entries = f.get_record_stream([(b'merged',)], 'unordered', False)
1235
merged = next(entries)
1236
delta_data = delta_adapter.get_bytes(merged)
1237
return ft_data, delta_data
1239
def test_deannotation_noeol(self):
1240
"""Test converting annotated knits to unannotated knits."""
1241
# we need a full text, and a delta
1243
get_diamond_files(f, 1, trailing_eol=False)
1244
ft_data, delta_data = self.helpGetBytes(f,
1245
_mod_knit.FTAnnotatedToUnannotated(None),
1246
_mod_knit.DeltaAnnotatedToUnannotated(None))
1248
b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1251
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1253
b'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1254
b'1,2,3\nleft\nright\nmerged\nend merged\n',
1255
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1257
def test_deannotation(self):
1258
"""Test converting annotated knits to unannotated knits."""
1259
# we need a full text, and a delta
1261
get_diamond_files(f, 1)
1262
ft_data, delta_data = self.helpGetBytes(f,
1263
_mod_knit.FTAnnotatedToUnannotated(None),
1264
_mod_knit.DeltaAnnotatedToUnannotated(None))
1266
b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1269
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1271
b'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1272
b'2,2,2\nright\nmerged\nend merged\n',
1273
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1275
def test_annotated_to_fulltext_no_eol(self):
1276
"""Test adapting annotated knits to full texts (for -> weaves)."""
1277
# we need a full text, and a delta
1279
get_diamond_files(f, 1, trailing_eol=False)
1280
# Reconstructing a full text requires a backing versioned file, and it
1281
# must have the base lines requested from it.
1282
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1283
ft_data, delta_data = self.helpGetBytes(f,
1284
_mod_knit.FTAnnotatedToFullText(None),
1285
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1286
self.assertEqual(b'origin', ft_data)
1287
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1288
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1289
True)], logged_vf.calls)
1291
def test_annotated_to_fulltext(self):
1292
"""Test adapting annotated knits to full texts (for -> weaves)."""
1293
# we need a full text, and a delta
1295
get_diamond_files(f, 1)
1296
# Reconstructing a full text requires a backing versioned file, and it
1297
# must have the base lines requested from it.
1298
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1299
ft_data, delta_data = self.helpGetBytes(f,
1300
_mod_knit.FTAnnotatedToFullText(None),
1301
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1302
self.assertEqual(b'origin\n', ft_data)
1303
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1304
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1305
True)], logged_vf.calls)
1307
def test_unannotated_to_fulltext(self):
1308
"""Test adapting unannotated knits to full texts.
1310
This is used for -> weaves, and for -> annotated knits.
1312
# we need a full text, and a delta
1313
f = self.get_knit(annotated=False)
1314
get_diamond_files(f, 1)
1315
# Reconstructing a full text requires a backing versioned file, and it
1316
# must have the base lines requested from it.
1317
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1318
ft_data, delta_data = self.helpGetBytes(f,
1319
_mod_knit.FTPlainToFullText(None),
1320
_mod_knit.DeltaPlainToFullText(logged_vf))
1321
self.assertEqual(b'origin\n', ft_data)
1322
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1323
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1324
True)], logged_vf.calls)
1326
def test_unannotated_to_fulltext_no_eol(self):
1327
"""Test adapting unannotated knits to full texts.
1329
This is used for -> weaves, and for -> annotated knits.
1331
# we need a full text, and a delta
1332
f = self.get_knit(annotated=False)
1333
get_diamond_files(f, 1, trailing_eol=False)
1334
# Reconstructing a full text requires a backing versioned file, and it
1335
# must have the base lines requested from it.
1336
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1337
ft_data, delta_data = self.helpGetBytes(f,
1338
_mod_knit.FTPlainToFullText(None),
1339
_mod_knit.DeltaPlainToFullText(logged_vf))
1340
self.assertEqual(b'origin', ft_data)
1341
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1342
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1343
True)], logged_vf.calls)
1346
class TestKeyMapper(TestCaseWithMemoryTransport):
1347
"""Tests for various key mapping logic."""
1349
def test_identity_mapper(self):
1350
mapper = versionedfile.ConstantMapper("inventory")
1351
self.assertEqual("inventory", mapper.map((b'foo@ar',)))
1352
self.assertEqual("inventory", mapper.map((b'quux',)))
1354
def test_prefix_mapper(self):
1356
mapper = versionedfile.PrefixMapper()
1357
self.assertEqual("file-id", mapper.map((b"file-id", b"revision-id")))
1358
self.assertEqual("new-id", mapper.map((b"new-id", b"revision-id")))
1359
self.assertEqual((b'file-id',), mapper.unmap("file-id"))
1360
self.assertEqual((b'new-id',), mapper.unmap("new-id"))
1362
def test_hash_prefix_mapper(self):
1363
#format6: hash + plain
1364
mapper = versionedfile.HashPrefixMapper()
1365
self.assertEqual("9b/file-id", mapper.map((b"file-id", b"revision-id")))
1366
self.assertEqual("45/new-id", mapper.map((b"new-id", b"revision-id")))
1367
self.assertEqual((b'file-id',), mapper.unmap("9b/file-id"))
1368
self.assertEqual((b'new-id',), mapper.unmap("45/new-id"))
1370
def test_hash_escaped_mapper(self):
1371
#knit1: hash + escaped
1372
mapper = versionedfile.HashEscapedPrefixMapper()
1373
self.assertEqual("88/%2520", mapper.map((b" ", b"revision-id")))
1374
self.assertEqual("ed/fil%2545-%2549d", mapper.map((b"filE-Id",
1376
self.assertEqual("88/ne%2557-%2549d", mapper.map((b"neW-Id",
1378
self.assertEqual((b'filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1379
self.assertEqual((b'neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1382
class TestVersionedFiles(TestCaseWithMemoryTransport):
1383
"""Tests for the multiple-file variant of VersionedFile."""
1385
# We want to be sure of behaviour for:
1386
# weaves prefix layout (weave texts)
1387
# individually named weaves (weave inventories)
1388
# annotated knits - prefix|hash|hash-escape layout, we test the third only
1389
# as it is the most complex mapper.
1390
# individually named knits
1391
# individual no-graph knits in packs (signatures)
1392
# individual graph knits in packs (inventories)
1393
# individual graph nocompression knits in packs (revisions)
1394
# plain text knits in packs (texts)
1395
len_one_scenarios = [
1398
'factory': make_versioned_files_factory(WeaveFile,
1399
ConstantMapper('inventory')),
1402
'support_partial_insertion': False,
1406
'factory': make_file_factory(False, ConstantMapper('revisions')),
1409
'support_partial_insertion': False,
1411
('named-nograph-nodelta-knit-pack', {
1412
'cleanup': cleanup_pack_knit,
1413
'factory': make_pack_factory(False, False, 1),
1416
'support_partial_insertion': False,
1418
('named-graph-knit-pack', {
1419
'cleanup': cleanup_pack_knit,
1420
'factory': make_pack_factory(True, True, 1),
1423
'support_partial_insertion': True,
1425
('named-graph-nodelta-knit-pack', {
1426
'cleanup': cleanup_pack_knit,
1427
'factory': make_pack_factory(True, False, 1),
1430
'support_partial_insertion': False,
1432
('groupcompress-nograph', {
1433
'cleanup': groupcompress.cleanup_pack_group,
1434
'factory': groupcompress.make_pack_factory(False, False, 1),
1437
'support_partial_insertion': False,
1440
len_two_scenarios = [
1443
'factory': make_versioned_files_factory(WeaveFile,
1447
'support_partial_insertion': False,
1449
('annotated-knit-escape', {
1451
'factory': make_file_factory(True, HashEscapedPrefixMapper()),
1454
'support_partial_insertion': False,
1456
('plain-knit-pack', {
1457
'cleanup': cleanup_pack_knit,
1458
'factory': make_pack_factory(True, True, 2),
1461
'support_partial_insertion': True,
1464
'cleanup': groupcompress.cleanup_pack_group,
1465
'factory': groupcompress.make_pack_factory(True, False, 1),
1468
'support_partial_insertion': False,
1472
scenarios = len_one_scenarios + len_two_scenarios
1474
def get_versionedfiles(self, relpath='files'):
1475
transport = self.get_transport(relpath)
1477
transport.mkdir('.')
1478
files = self.factory(transport)
1479
if self.cleanup is not None:
1480
self.addCleanup(self.cleanup, files)
1483
def get_simple_key(self, suffix):
1484
"""Return a key for the object under test."""
1485
if self.key_length == 1:
1488
return (b'FileA',) + (suffix,)
1490
def test_add_fallback_implies_without_fallbacks(self):
1491
f = self.get_versionedfiles('files')
1492
if getattr(f, 'add_fallback_versioned_files', None) is None:
1493
raise TestNotApplicable("%s doesn't support fallbacks"
1494
% (f.__class__.__name__,))
1495
g = self.get_versionedfiles('fallback')
1496
key_a = self.get_simple_key(b'a')
1497
g.add_lines(key_a, [], [b'\n'])
1498
f.add_fallback_versioned_files(g)
1499
self.assertTrue(key_a in f.get_parent_map([key_a]))
1500
self.assertFalse(key_a in f.without_fallbacks().get_parent_map([key_a]))
1502
def test_add_lines(self):
1503
f = self.get_versionedfiles()
1504
key0 = self.get_simple_key(b'r0')
1505
key1 = self.get_simple_key(b'r1')
1506
key2 = self.get_simple_key(b'r2')
1507
keyf = self.get_simple_key(b'foo')
1508
f.add_lines(key0, [], [b'a\n', b'b\n'])
1510
f.add_lines(key1, [key0], [b'b\n', b'c\n'])
1512
f.add_lines(key1, [], [b'b\n', b'c\n'])
1514
self.assertTrue(key0 in keys)
1515
self.assertTrue(key1 in keys)
1517
for record in f.get_record_stream([key0, key1], 'unordered', True):
1518
records.append((record.key, record.get_bytes_as('fulltext')))
1520
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1522
def test_annotate(self):
1523
files = self.get_versionedfiles()
1524
self.get_diamond_files(files)
1525
if self.key_length == 1:
1528
prefix = (b'FileA',)
1529
# introduced full text
1530
origins = files.annotate(prefix + (b'origin',))
1532
(prefix + (b'origin',), b'origin\n')],
1535
origins = files.annotate(prefix + (b'base',))
1537
(prefix + (b'base',), b'base\n')],
1540
origins = files.annotate(prefix + (b'merged',))
1543
(prefix + (b'base',), b'base\n'),
1544
(prefix + (b'left',), b'left\n'),
1545
(prefix + (b'right',), b'right\n'),
1546
(prefix + (b'merged',), b'merged\n')
1550
# Without a graph everything is new.
1552
(prefix + (b'merged',), b'base\n'),
1553
(prefix + (b'merged',), b'left\n'),
1554
(prefix + (b'merged',), b'right\n'),
1555
(prefix + (b'merged',), b'merged\n')
1558
self.assertRaises(RevisionNotPresent,
1559
files.annotate, prefix + ('missing-key',))
1561
def test_check_no_parameters(self):
1562
files = self.get_versionedfiles()
1564
def test_check_progressbar_parameter(self):
1565
"""A progress bar can be supplied because check can be a generator."""
1566
pb = ui.ui_factory.nested_progress_bar()
1567
self.addCleanup(pb.finished)
1568
files = self.get_versionedfiles()
1569
files.check(progress_bar=pb)
1571
def test_check_with_keys_becomes_generator(self):
1572
files = self.get_versionedfiles()
1573
self.get_diamond_files(files)
1575
entries = files.check(keys=keys)
1577
# Texts output should be fulltexts.
1578
self.capture_stream(files, entries, seen.add,
1579
files.get_parent_map(keys), require_fulltext=True)
1580
# All texts should be output.
1581
self.assertEqual(set(keys), seen)
1583
def test_clear_cache(self):
1584
files = self.get_versionedfiles()
1587
def test_construct(self):
1588
"""Each parameterised test can be constructed on a transport."""
1589
files = self.get_versionedfiles()
1591
def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1593
return get_diamond_files(files, self.key_length,
1594
trailing_eol=trailing_eol, nograph=not self.graph,
1595
left_only=left_only, nokeys=nokeys)
1597
def _add_content_nostoresha(self, add_lines):
1598
"""When nostore_sha is supplied using old content raises."""
1599
vf = self.get_versionedfiles()
1600
empty_text = (b'a', [])
1601
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
1602
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
1604
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1606
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1609
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1612
# we now have a copy of all the lines in the vf.
1613
for sha, (version, lines) in zip(
1614
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1615
new_key = self.get_simple_key(version + b"2")
1616
self.assertRaises(errors.ExistingContent,
1617
vf.add_lines, new_key, [], lines,
1619
self.assertRaises(errors.ExistingContent,
1620
vf.add_lines, new_key, [], lines,
1622
# and no new version should have been added.
1623
record = next(vf.get_record_stream([new_key], 'unordered', True))
1624
self.assertEqual('absent', record.storage_kind)
1626
def test_add_lines_nostoresha(self):
1627
self._add_content_nostoresha(add_lines=True)
1629
def test_add_lines_return(self):
1630
files = self.get_versionedfiles()
1631
# save code by using the stock data insertion helper.
1632
adds = self.get_diamond_files(files)
1634
# We can only validate the first 2 elements returned from add_lines.
1636
self.assertEqual(3, len(add))
1637
results.append(add[:2])
1638
if self.key_length == 1:
1640
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1641
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1642
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1643
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1644
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1646
elif self.key_length == 2:
1648
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1649
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1650
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1651
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1652
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1653
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1654
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1655
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1656
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1657
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1660
def test_add_lines_no_key_generates_chk_key(self):
1661
files = self.get_versionedfiles()
1662
# save code by using the stock data insertion helper.
1663
adds = self.get_diamond_files(files, nokeys=True)
1665
# We can only validate the first 2 elements returned from add_lines.
1667
self.assertEqual(3, len(add))
1668
results.append(add[:2])
1669
if self.key_length == 1:
1671
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1672
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1673
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1674
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1675
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1677
# Check the added items got CHK keys.
1679
(b'sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1680
(b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1681
(b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1682
(b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1683
(b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1686
elif self.key_length == 2:
1688
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1689
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1690
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1691
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1692
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1693
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1694
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1695
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1696
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1697
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1699
# Check the added items got CHK keys.
1701
(b'FileA', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1702
(b'FileA', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1703
(b'FileA', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1704
(b'FileA', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1705
(b'FileA', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1706
(b'FileB', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1707
(b'FileB', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1708
(b'FileB', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1709
(b'FileB', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1710
(b'FileB', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1714
def test_empty_lines(self):
1715
"""Empty files can be stored."""
1716
f = self.get_versionedfiles()
1717
key_a = self.get_simple_key(b'a')
1718
f.add_lines(key_a, [], [])
1719
self.assertEqual(b'',
1720
next(f.get_record_stream([key_a], 'unordered', True
1721
)).get_bytes_as('fulltext'))
1722
key_b = self.get_simple_key(b'b')
1723
f.add_lines(key_b, self.get_parents([key_a]), [])
1724
self.assertEqual(b'',
1725
next(f.get_record_stream([key_b], 'unordered', True
1726
)).get_bytes_as('fulltext'))
1728
def test_newline_only(self):
1729
f = self.get_versionedfiles()
1730
key_a = self.get_simple_key(b'a')
1731
f.add_lines(key_a, [], [b'\n'])
1732
self.assertEqual(b'\n',
1733
next(f.get_record_stream([key_a], 'unordered', True
1734
)).get_bytes_as('fulltext'))
1735
key_b = self.get_simple_key(b'b')
1736
f.add_lines(key_b, self.get_parents([key_a]), [b'\n'])
1737
self.assertEqual(b'\n',
1738
next(f.get_record_stream([key_b], 'unordered', True
1739
)).get_bytes_as('fulltext'))
1741
def test_get_known_graph_ancestry(self):
1742
f = self.get_versionedfiles()
1744
raise TestNotApplicable('ancestry info only relevant with graph.')
1745
key_a = self.get_simple_key(b'a')
1746
key_b = self.get_simple_key(b'b')
1747
key_c = self.get_simple_key(b'c')
1753
f.add_lines(key_a, [], [b'\n'])
1754
f.add_lines(key_b, [key_a], [b'\n'])
1755
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1756
kg = f.get_known_graph_ancestry([key_c])
1757
self.assertIsInstance(kg, _mod_graph.KnownGraph)
1758
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1760
def test_known_graph_with_fallbacks(self):
1761
f = self.get_versionedfiles('files')
1763
raise TestNotApplicable('ancestry info only relevant with graph.')
1764
if getattr(f, 'add_fallback_versioned_files', None) is None:
1765
raise TestNotApplicable("%s doesn't support fallbacks"
1766
% (f.__class__.__name__,))
1767
key_a = self.get_simple_key(b'a')
1768
key_b = self.get_simple_key(b'b')
1769
key_c = self.get_simple_key(b'c')
1770
# A only in fallback
1775
g = self.get_versionedfiles('fallback')
1776
g.add_lines(key_a, [], [b'\n'])
1777
f.add_fallback_versioned_files(g)
1778
f.add_lines(key_b, [key_a], [b'\n'])
1779
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1780
kg = f.get_known_graph_ancestry([key_c])
1781
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1783
def test_get_record_stream_empty(self):
1784
"""An empty stream can be requested without error."""
1785
f = self.get_versionedfiles()
1786
entries = f.get_record_stream([], 'unordered', False)
1787
self.assertEqual([], list(entries))
1789
def assertValidStorageKind(self, storage_kind):
1790
"""Assert that storage_kind is a valid storage_kind."""
1791
self.assertSubset([storage_kind],
1792
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1793
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1794
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1796
'knit-delta-closure', 'knit-delta-closure-ref',
1797
'groupcompress-block', 'groupcompress-block-ref'])
1799
def capture_stream(self, f, entries, on_seen, parents,
1800
require_fulltext=False):
1801
"""Capture a stream for testing."""
1802
for factory in entries:
1803
on_seen(factory.key)
1804
self.assertValidStorageKind(factory.storage_kind)
1805
if factory.sha1 is not None:
1806
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1808
self.assertEqual(parents[factory.key], factory.parents)
1809
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1811
if require_fulltext:
1812
factory.get_bytes_as('fulltext')
1814
def test_get_record_stream_interface(self):
1815
"""each item in a stream has to provide a regular interface."""
1816
files = self.get_versionedfiles()
1817
self.get_diamond_files(files)
1818
keys, _ = self.get_keys_and_sort_order()
1819
parent_map = files.get_parent_map(keys)
1820
entries = files.get_record_stream(keys, 'unordered', False)
1822
self.capture_stream(files, entries, seen.add, parent_map)
1823
self.assertEqual(set(keys), seen)
1825
def get_keys_and_sort_order(self):
1826
"""Get diamond test keys list, and their sort ordering."""
1827
if self.key_length == 1:
1828
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1829
sort_order = {(b'merged',):2, (b'left',):1, (b'right',):1, (b'base',):0}
1832
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1833
(b'FileA', b'base'),
1834
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1835
(b'FileB', b'base'),
1838
(b'FileA', b'merged'): 2, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1839
(b'FileA', b'base'): 0,
1840
(b'FileB', b'merged'): 2, (b'FileB', b'left'): 1, (b'FileB', b'right'): 1,
1841
(b'FileB', b'base'): 0,
1843
return keys, sort_order
1845
def get_keys_and_groupcompress_sort_order(self):
1846
"""Get diamond test keys list, and their groupcompress sort ordering."""
1847
if self.key_length == 1:
1848
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1849
sort_order = {(b'merged',): 0, (b'left',): 1, (b'right',): 1, (b'base',):2}
1852
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1853
(b'FileA', b'base'),
1854
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1855
(b'FileB', b'base'),
1858
(b'FileA', b'merged'): 0, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1859
(b'FileA', b'base'): 2,
1860
(b'FileB', b'merged'): 3, (b'FileB', b'left'): 4, (b'FileB', b'right'): 4,
1861
(b'FileB', b'base'): 5,
1863
return keys, sort_order
1865
def test_get_record_stream_interface_ordered(self):
1866
"""each item in a stream has to provide a regular interface."""
1867
files = self.get_versionedfiles()
1868
self.get_diamond_files(files)
1869
keys, sort_order = self.get_keys_and_sort_order()
1870
parent_map = files.get_parent_map(keys)
1871
entries = files.get_record_stream(keys, 'topological', False)
1873
self.capture_stream(files, entries, seen.append, parent_map)
1874
self.assertStreamOrder(sort_order, seen, keys)
1876
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1877
"""each item must be accessible as a fulltext."""
1878
files = self.get_versionedfiles()
1879
self.get_diamond_files(files)
1880
keys, sort_order = self.get_keys_and_sort_order()
1881
parent_map = files.get_parent_map(keys)
1882
entries = files.get_record_stream(keys, 'topological', True)
1884
for factory in entries:
1885
seen.append(factory.key)
1886
self.assertValidStorageKind(factory.storage_kind)
1887
self.assertSubset([factory.sha1],
1888
[None, files.get_sha1s([factory.key])[factory.key]])
1889
self.assertEqual(parent_map[factory.key], factory.parents)
1890
# self.assertEqual(files.get_text(factory.key),
1891
ft_bytes = factory.get_bytes_as('fulltext')
1892
self.assertIsInstance(ft_bytes, bytes)
1893
chunked_bytes = factory.get_bytes_as('chunked')
1894
self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1896
self.assertStreamOrder(sort_order, seen, keys)
1898
def test_get_record_stream_interface_groupcompress(self):
1899
"""each item in a stream has to provide a regular interface."""
1900
files = self.get_versionedfiles()
1901
self.get_diamond_files(files)
1902
keys, sort_order = self.get_keys_and_groupcompress_sort_order()
1903
parent_map = files.get_parent_map(keys)
1904
entries = files.get_record_stream(keys, 'groupcompress', False)
1906
self.capture_stream(files, entries, seen.append, parent_map)
1907
self.assertStreamOrder(sort_order, seen, keys)
1909
def assertStreamOrder(self, sort_order, seen, keys):
1910
self.assertEqual(len(set(seen)), len(keys))
1911
if self.key_length == 1:
1914
lows = {(b'FileA',):0, (b'FileB',):0}
1916
self.assertEqual(set(keys), set(seen))
1919
sort_pos = sort_order[key]
1920
self.assertTrue(sort_pos >= lows[key[:-1]],
1921
"Out of order in sorted stream: %r, %r" % (key, seen))
1922
lows[key[:-1]] = sort_pos
1924
def test_get_record_stream_unknown_storage_kind_raises(self):
1925
"""Asking for a storage kind that the stream cannot supply raises."""
1926
files = self.get_versionedfiles()
1927
self.get_diamond_files(files)
1928
if self.key_length == 1:
1929
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1932
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1933
(b'FileA', b'base'),
1934
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1935
(b'FileB', b'base'),
1937
parent_map = files.get_parent_map(keys)
1938
entries = files.get_record_stream(keys, 'unordered', False)
1939
# We track the contents because we should be able to try, fail a
1940
# particular kind and then ask for one that works and continue.
1942
for factory in entries:
1943
seen.add(factory.key)
1944
self.assertValidStorageKind(factory.storage_kind)
1945
if factory.sha1 is not None:
1946
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1948
self.assertEqual(parent_map[factory.key], factory.parents)
1949
# currently no stream emits mpdiff
1950
self.assertRaises(errors.UnavailableRepresentation,
1951
factory.get_bytes_as, 'mpdiff')
1952
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1954
self.assertEqual(set(keys), seen)
1956
def test_get_record_stream_missing_records_are_absent(self):
1957
files = self.get_versionedfiles()
1958
self.get_diamond_files(files)
1959
if self.key_length == 1:
1960
keys = [(b'merged',), (b'left',), (b'right',), (b'absent',), (b'base',)]
1963
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1964
(b'FileA', b'absent'), (b'FileA', b'base'),
1965
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1966
(b'FileB', b'absent'), (b'FileB', b'base'),
1967
(b'absent', b'absent'),
1969
parent_map = files.get_parent_map(keys)
1970
entries = files.get_record_stream(keys, 'unordered', False)
1971
self.assertAbsentRecord(files, keys, parent_map, entries)
1972
entries = files.get_record_stream(keys, 'topological', False)
1973
self.assertAbsentRecord(files, keys, parent_map, entries)
1975
def assertRecordHasContent(self, record, bytes):
1976
"""Assert that record has the bytes bytes."""
1977
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
1978
self.assertEqual(bytes, b''.join(record.get_bytes_as('chunked')))
1980
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
1981
files = self.get_versionedfiles()
1982
key = self.get_simple_key(b'foo')
1983
files.add_lines(key, (), [b'my text\n', b'content'])
1984
stream = files.get_record_stream([key], 'unordered', False)
1985
record = next(stream)
1986
if record.storage_kind in ('chunked', 'fulltext'):
1987
# chunked and fulltext representations are for direct use not wire
1988
# serialisation: check they are able to be used directly. To send
1989
# such records over the wire translation will be needed.
1990
self.assertRecordHasContent(record, b"my text\ncontent")
1992
bytes = [record.get_bytes_as(record.storage_kind)]
1993
network_stream = versionedfile.NetworkRecordStream(bytes).read()
1994
source_record = record
1996
for record in network_stream:
1997
records.append(record)
1998
self.assertEqual(source_record.storage_kind,
1999
record.storage_kind)
2000
self.assertEqual(source_record.parents, record.parents)
2002
source_record.get_bytes_as(source_record.storage_kind),
2003
record.get_bytes_as(record.storage_kind))
2004
self.assertEqual(1, len(records))
2006
def assertStreamMetaEqual(self, records, expected, stream):
2007
"""Assert that streams expected and stream have the same records.
2009
:param records: A list to collect the seen records.
2010
:return: A generator of the records in stream.
2012
# We make assertions during copying to catch things early for easier
2013
# debugging. This must use the iterating zip() from the future.
2014
for record, ref_record in zip(stream, expected):
2015
records.append(record)
2016
self.assertEqual(ref_record.key, record.key)
2017
self.assertEqual(ref_record.storage_kind, record.storage_kind)
2018
self.assertEqual(ref_record.parents, record.parents)
2021
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
2023
"""Convert a stream to a bytes iterator.
2025
:param skipped_records: A list with one element to increment when a
2027
:param full_texts: A dict from key->fulltext representation, for
2028
checking chunked or fulltext stored records.
2029
:param stream: A record_stream.
2030
:return: An iterator over the bytes of each record.
2032
for record in stream:
2033
if record.storage_kind in ('chunked', 'fulltext'):
2034
skipped_records[0] += 1
2035
# check the content is correct for direct use.
2036
self.assertRecordHasContent(record, full_texts[record.key])
2038
yield record.get_bytes_as(record.storage_kind)
2040
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
2041
files = self.get_versionedfiles()
2042
target_files = self.get_versionedfiles('target')
2043
key = self.get_simple_key(b'ft')
2044
key_delta = self.get_simple_key(b'delta')
2045
files.add_lines(key, (), [b'my text\n', b'content'])
2047
delta_parents = (key,)
2050
files.add_lines(key_delta, delta_parents, [b'different\n', b'content\n'])
2051
local = files.get_record_stream([key, key_delta], 'unordered', False)
2052
ref = files.get_record_stream([key, key_delta], 'unordered', False)
2053
skipped_records = [0]
2055
key: b"my text\ncontent",
2056
key_delta: b"different\ncontent\n",
2058
byte_stream = self.stream_to_bytes_or_skip_counter(
2059
skipped_records, full_texts, local)
2060
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2062
# insert the stream from the network into a versioned files object so we can
2063
# check the content was carried across correctly without doing delta
2065
target_files.insert_record_stream(
2066
self.assertStreamMetaEqual(records, ref, network_stream))
2067
# No duplicates on the wire thank you!
2068
self.assertEqual(2, len(records) + skipped_records[0])
2070
# if any content was copied it all must have all been.
2071
self.assertIdenticalVersionedFile(files, target_files)
2073
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
2074
# copy a delta over the wire
2075
files = self.get_versionedfiles()
2076
target_files = self.get_versionedfiles('target')
2077
key = self.get_simple_key(b'ft')
2078
key_delta = self.get_simple_key(b'delta')
2079
files.add_lines(key, (), [b'my text\n', b'content'])
2081
delta_parents = (key,)
2084
files.add_lines(key_delta, delta_parents, [b'different\n', b'content\n'])
2085
# Copy the basis text across so we can reconstruct the delta during
2086
# insertion into target.
2087
target_files.insert_record_stream(files.get_record_stream([key],
2088
'unordered', False))
2089
local = files.get_record_stream([key_delta], 'unordered', False)
2090
ref = files.get_record_stream([key_delta], 'unordered', False)
2091
skipped_records = [0]
2093
key_delta: b"different\ncontent\n",
2095
byte_stream = self.stream_to_bytes_or_skip_counter(
2096
skipped_records, full_texts, local)
2097
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2099
# insert the stream from the network into a versioned files object so we can
2100
# check the content was carried across correctly without doing delta
2101
# inspection during check_stream.
2102
target_files.insert_record_stream(
2103
self.assertStreamMetaEqual(records, ref, network_stream))
2104
# No duplicates on the wire thank you!
2105
self.assertEqual(1, len(records) + skipped_records[0])
2107
# if any content was copied it all must have all been
2108
self.assertIdenticalVersionedFile(files, target_files)
2110
def test_get_record_stream_wire_ready_delta_closure_included(self):
2111
# copy a delta over the wire with the ability to get its full text.
2112
files = self.get_versionedfiles()
2113
key = self.get_simple_key(b'ft')
2114
key_delta = self.get_simple_key(b'delta')
2115
files.add_lines(key, (), [b'my text\n', b'content'])
2117
delta_parents = (key,)
2120
files.add_lines(key_delta, delta_parents, [b'different\n', b'content\n'])
2121
local = files.get_record_stream([key_delta], 'unordered', True)
2122
ref = files.get_record_stream([key_delta], 'unordered', True)
2123
skipped_records = [0]
2125
key_delta: b"different\ncontent\n",
2127
byte_stream = self.stream_to_bytes_or_skip_counter(
2128
skipped_records, full_texts, local)
2129
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2131
# insert the stream from the network into a versioned files object so we can
2132
# check the content was carried across correctly without doing delta
2133
# inspection during check_stream.
2134
for record in self.assertStreamMetaEqual(records, ref, network_stream):
2135
# we have to be able to get the full text out:
2136
self.assertRecordHasContent(record, full_texts[record.key])
2137
# No duplicates on the wire thank you!
2138
self.assertEqual(1, len(records) + skipped_records[0])
2140
def assertAbsentRecord(self, files, keys, parents, entries):
2141
"""Helper for test_get_record_stream_missing_records_are_absent."""
2143
for factory in entries:
2144
seen.add(factory.key)
2145
if factory.key[-1] == b'absent':
2146
self.assertEqual('absent', factory.storage_kind)
2147
self.assertEqual(None, factory.sha1)
2148
self.assertEqual(None, factory.parents)
2150
self.assertValidStorageKind(factory.storage_kind)
2151
if factory.sha1 is not None:
2152
sha1 = files.get_sha1s([factory.key])[factory.key]
2153
self.assertEqual(sha1, factory.sha1)
2154
self.assertEqual(parents[factory.key], factory.parents)
2155
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2157
self.assertEqual(set(keys), seen)
2159
def test_filter_absent_records(self):
2160
"""Requested missing records can be filter trivially."""
2161
files = self.get_versionedfiles()
2162
self.get_diamond_files(files)
2163
keys, _ = self.get_keys_and_sort_order()
2164
parent_map = files.get_parent_map(keys)
2165
# Add an absent record in the middle of the present keys. (We don't ask
2166
# for just absent keys to ensure that content before and after the
2167
# absent keys is still delivered).
2168
present_keys = list(keys)
2169
if self.key_length == 1:
2170
keys.insert(2, (b'extra',))
2172
keys.insert(2, (b'extra', b'extra'))
2173
entries = files.get_record_stream(keys, 'unordered', False)
2175
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2177
self.assertEqual(set(present_keys), seen)
2179
def get_mapper(self):
2180
"""Get a mapper suitable for the key length of the test interface."""
2181
if self.key_length == 1:
2182
return ConstantMapper('source')
2184
return HashEscapedPrefixMapper()
2186
def get_parents(self, parents):
2187
"""Get parents, taking self.graph into consideration."""
2193
def test_get_annotator(self):
2194
files = self.get_versionedfiles()
2195
self.get_diamond_files(files)
2196
origin_key = self.get_simple_key(b'origin')
2197
base_key = self.get_simple_key(b'base')
2198
left_key = self.get_simple_key(b'left')
2199
right_key = self.get_simple_key(b'right')
2200
merged_key = self.get_simple_key(b'merged')
2201
# annotator = files.get_annotator()
2202
# introduced full text
2203
origins, lines = files.get_annotator().annotate(origin_key)
2204
self.assertEqual([(origin_key,)], origins)
2205
self.assertEqual([b'origin\n'], lines)
2207
origins, lines = files.get_annotator().annotate(base_key)
2208
self.assertEqual([(base_key,)], origins)
2210
origins, lines = files.get_annotator().annotate(merged_key)
2219
# Without a graph everything is new.
2226
self.assertRaises(RevisionNotPresent,
2227
files.get_annotator().annotate, self.get_simple_key(b'missing-key'))
2229
def test_get_parent_map(self):
2230
files = self.get_versionedfiles()
2231
if self.key_length == 1:
2233
((b'r0',), self.get_parents(())),
2234
((b'r1',), self.get_parents(((b'r0',),))),
2235
((b'r2',), self.get_parents(())),
2236
((b'r3',), self.get_parents(())),
2237
((b'm',), self.get_parents(((b'r0',), (b'r1',), (b'r2',), (b'r3',)))),
2241
((b'FileA', b'r0'), self.get_parents(())),
2242
((b'FileA', b'r1'), self.get_parents(((b'FileA', b'r0'),))),
2243
((b'FileA', b'r2'), self.get_parents(())),
2244
((b'FileA', b'r3'), self.get_parents(())),
2245
((b'FileA', b'm'), self.get_parents(((b'FileA', b'r0'),
2246
(b'FileA', b'r1'), (b'FileA', b'r2'), (b'FileA', b'r3')))),
2248
for key, parents in parent_details:
2249
files.add_lines(key, parents, [])
2250
# immediately after adding it should be queryable.
2251
self.assertEqual({key:parents}, files.get_parent_map([key]))
2252
# We can ask for an empty set
2253
self.assertEqual({}, files.get_parent_map([]))
2254
# We can ask for many keys
2255
all_parents = dict(parent_details)
2256
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2257
# Absent keys are just not included in the result.
2258
keys = list(all_parents.keys())
2259
if self.key_length == 1:
2260
keys.insert(1, (b'missing',))
2262
keys.insert(1, (b'missing', b'missing'))
2263
# Absent keys are just ignored
2264
self.assertEqual(all_parents, files.get_parent_map(keys))
2266
def test_get_sha1s(self):
2267
files = self.get_versionedfiles()
2268
self.get_diamond_files(files)
2269
if self.key_length == 1:
2270
keys = [(b'base',), (b'origin',), (b'left',), (b'merged',), (b'right',)]
2272
# ask for shas from different prefixes.
2274
(b'FileA', b'base'), (b'FileB', b'origin'), (b'FileA', b'left'),
2275
(b'FileA', b'merged'), (b'FileB', b'right'),
2278
keys[0]: b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2279
keys[1]: b'00e364d235126be43292ab09cb4686cf703ddc17',
2280
keys[2]: b'a8478686da38e370e32e42e8a0c220e33ee9132f',
2281
keys[3]: b'ed8bce375198ea62444dc71952b22cfc2b09226d',
2282
keys[4]: b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2284
files.get_sha1s(keys))
2286
def test_insert_record_stream_empty(self):
2287
"""Inserting an empty record stream should work."""
2288
files = self.get_versionedfiles()
2289
files.insert_record_stream([])
2291
def assertIdenticalVersionedFile(self, expected, actual):
2292
"""Assert that left and right have the same contents."""
2293
self.assertEqual(set(actual.keys()), set(expected.keys()))
2294
actual_parents = actual.get_parent_map(actual.keys())
2296
self.assertEqual(actual_parents, expected.get_parent_map(expected.keys()))
2298
for key, parents in actual_parents.items():
2299
self.assertEqual(None, parents)
2300
for key in actual.keys():
2301
actual_text = next(actual.get_record_stream(
2302
[key], 'unordered', True)).get_bytes_as('fulltext')
2303
expected_text = next(expected.get_record_stream(
2304
[key], 'unordered', True)).get_bytes_as('fulltext')
2305
self.assertEqual(actual_text, expected_text)
2307
def test_insert_record_stream_fulltexts(self):
2308
"""Any file should accept a stream of fulltexts."""
2309
files = self.get_versionedfiles()
2310
mapper = self.get_mapper()
2311
source_transport = self.get_transport('source')
2312
source_transport.mkdir('.')
2313
# weaves always output fulltexts.
2314
source = make_versioned_files_factory(WeaveFile, mapper)(
2316
self.get_diamond_files(source, trailing_eol=False)
2317
stream = source.get_record_stream(source.keys(), 'topological',
2319
files.insert_record_stream(stream)
2320
self.assertIdenticalVersionedFile(source, files)
2322
def test_insert_record_stream_fulltexts_noeol(self):
2323
"""Any file should accept a stream of fulltexts."""
2324
files = self.get_versionedfiles()
2325
mapper = self.get_mapper()
2326
source_transport = self.get_transport('source')
2327
source_transport.mkdir('.')
2328
# weaves always output fulltexts.
2329
source = make_versioned_files_factory(WeaveFile, mapper)(
2331
self.get_diamond_files(source, trailing_eol=False)
2332
stream = source.get_record_stream(source.keys(), 'topological',
2334
files.insert_record_stream(stream)
2335
self.assertIdenticalVersionedFile(source, files)
2337
def test_insert_record_stream_annotated_knits(self):
2338
"""Any file should accept a stream from plain knits."""
2339
files = self.get_versionedfiles()
2340
mapper = self.get_mapper()
2341
source_transport = self.get_transport('source')
2342
source_transport.mkdir('.')
2343
source = make_file_factory(True, mapper)(source_transport)
2344
self.get_diamond_files(source)
2345
stream = source.get_record_stream(source.keys(), 'topological',
2347
files.insert_record_stream(stream)
2348
self.assertIdenticalVersionedFile(source, files)
2350
def test_insert_record_stream_annotated_knits_noeol(self):
2351
"""Any file should accept a stream from plain knits."""
2352
files = self.get_versionedfiles()
2353
mapper = self.get_mapper()
2354
source_transport = self.get_transport('source')
2355
source_transport.mkdir('.')
2356
source = make_file_factory(True, mapper)(source_transport)
2357
self.get_diamond_files(source, trailing_eol=False)
2358
stream = source.get_record_stream(source.keys(), 'topological',
2360
files.insert_record_stream(stream)
2361
self.assertIdenticalVersionedFile(source, files)
2363
def test_insert_record_stream_plain_knits(self):
2364
"""Any file should accept a stream from plain knits."""
2365
files = self.get_versionedfiles()
2366
mapper = self.get_mapper()
2367
source_transport = self.get_transport('source')
2368
source_transport.mkdir('.')
2369
source = make_file_factory(False, mapper)(source_transport)
2370
self.get_diamond_files(source)
2371
stream = source.get_record_stream(source.keys(), 'topological',
2373
files.insert_record_stream(stream)
2374
self.assertIdenticalVersionedFile(source, files)
2376
def test_insert_record_stream_plain_knits_noeol(self):
2377
"""Any file should accept a stream from plain knits."""
2378
files = self.get_versionedfiles()
2379
mapper = self.get_mapper()
2380
source_transport = self.get_transport('source')
2381
source_transport.mkdir('.')
2382
source = make_file_factory(False, mapper)(source_transport)
2383
self.get_diamond_files(source, trailing_eol=False)
2384
stream = source.get_record_stream(source.keys(), 'topological',
2386
files.insert_record_stream(stream)
2387
self.assertIdenticalVersionedFile(source, files)
2389
def test_insert_record_stream_existing_keys(self):
2390
"""Inserting keys already in a file should not error."""
2391
files = self.get_versionedfiles()
2392
source = self.get_versionedfiles('source')
2393
self.get_diamond_files(source)
2394
# insert some keys into f.
2395
self.get_diamond_files(files, left_only=True)
2396
stream = source.get_record_stream(source.keys(), 'topological',
2398
files.insert_record_stream(stream)
2399
self.assertIdenticalVersionedFile(source, files)
2401
def test_insert_record_stream_missing_keys(self):
2402
"""Inserting a stream with absent keys should raise an error."""
2403
files = self.get_versionedfiles()
2404
source = self.get_versionedfiles('source')
2405
stream = source.get_record_stream([(b'missing',) * self.key_length],
2406
'topological', False)
2407
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2410
def test_insert_record_stream_out_of_order(self):
2411
"""An out of order stream can either error or work."""
2412
files = self.get_versionedfiles()
2413
source = self.get_versionedfiles('source')
2414
self.get_diamond_files(source)
2415
if self.key_length == 1:
2416
origin_keys = [(b'origin',)]
2417
end_keys = [(b'merged',), (b'left',)]
2418
start_keys = [(b'right',), (b'base',)]
2420
origin_keys = [(b'FileA', b'origin'), (b'FileB', b'origin')]
2421
end_keys = [(b'FileA', b'merged',), (b'FileA', b'left',),
2422
(b'FileB', b'merged',), (b'FileB', b'left',)]
2423
start_keys = [(b'FileA', b'right',), (b'FileA', b'base',),
2424
(b'FileB', b'right',), (b'FileB', b'base',)]
2425
origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
2426
end_entries = source.get_record_stream(end_keys, 'topological', False)
2427
start_entries = source.get_record_stream(start_keys, 'topological', False)
2428
entries = itertools.chain(origin_entries, end_entries, start_entries)
2430
files.insert_record_stream(entries)
2431
except RevisionNotPresent:
2432
# Must not have corrupted the file.
2435
self.assertIdenticalVersionedFile(source, files)
2437
def test_insert_record_stream_long_parent_chain_out_of_order(self):
2438
"""An out of order stream can either error or work."""
2440
raise TestNotApplicable('ancestry info only relevant with graph.')
2441
# Create a reasonably long chain of records based on each other, where
2442
# most will be deltas.
2443
source = self.get_versionedfiles('source')
2446
content = [(b'same same %d\n' % n) for n in range(500)]
2447
letters = b'abcdefghijklmnopqrstuvwxyz'
2448
for i in range(len(letters)):
2449
letter = letters[i:i+1]
2450
key = (b'key-' + letter,)
2451
if self.key_length == 2:
2452
key = (b'prefix',) + key
2453
content.append(b'content for ' + letter + b'\n')
2454
source.add_lines(key, parents, content)
2457
# Create a stream of these records, excluding the first record that the
2458
# rest ultimately depend upon, and insert it into a new vf.
2460
for key in reversed(keys):
2461
streams.append(source.get_record_stream([key], 'unordered', False))
2462
deltas = itertools.chain.from_iterable(streams[:-1])
2463
files = self.get_versionedfiles()
2465
files.insert_record_stream(deltas)
2466
except RevisionNotPresent:
2467
# Must not have corrupted the file.
2470
# Must only report either just the first key as a missing parent,
2471
# no key as missing (for nodelta scenarios).
2472
missing = set(files.get_missing_compression_parent_keys())
2473
missing.discard(keys[0])
2474
self.assertEqual(set(), missing)
2476
def get_knit_delta_source(self):
2477
"""Get a source that can produce a stream with knit delta records,
2478
regardless of this test's scenario.
2480
mapper = self.get_mapper()
2481
source_transport = self.get_transport('source')
2482
source_transport.mkdir('.')
2483
source = make_file_factory(False, mapper)(source_transport)
2484
get_diamond_files(source, self.key_length, trailing_eol=True,
2485
nograph=False, left_only=False)
2488
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2489
"""Insertion where a needed basis is not included notifies the caller
2490
of the missing basis. In the meantime a record missing its basis is
2493
source = self.get_knit_delta_source()
2494
keys = [self.get_simple_key(b'origin'), self.get_simple_key(b'merged')]
2495
entries = source.get_record_stream(keys, 'unordered', False)
2496
files = self.get_versionedfiles()
2497
if self.support_partial_insertion:
2498
self.assertEqual([],
2499
list(files.get_missing_compression_parent_keys()))
2500
files.insert_record_stream(entries)
2501
missing_bases = files.get_missing_compression_parent_keys()
2502
self.assertEqual({self.get_simple_key(b'left')},
2504
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2507
errors.RevisionNotPresent, files.insert_record_stream, entries)
2510
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2511
"""Insertion where a needed basis is not included notifies the caller
2512
of the missing basis. That basis can be added in a second
2513
insert_record_stream call that does not need to repeat records present
2514
in the previous stream. The record(s) that required that basis are
2515
fully inserted once their basis is no longer missing.
2517
if not self.support_partial_insertion:
2518
raise TestNotApplicable(
2519
'versioned file scenario does not support partial insertion')
2520
source = self.get_knit_delta_source()
2521
entries = source.get_record_stream([self.get_simple_key(b'origin'),
2522
self.get_simple_key(b'merged')], 'unordered', False)
2523
files = self.get_versionedfiles()
2524
files.insert_record_stream(entries)
2525
missing_bases = files.get_missing_compression_parent_keys()
2526
self.assertEqual({self.get_simple_key(b'left')},
2528
# 'merged' is inserted (although a commit of a write group involving
2529
# this versionedfiles would fail).
2530
merged_key = self.get_simple_key(b'merged')
2532
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2533
# Add the full delta closure of the missing records
2534
missing_entries = source.get_record_stream(
2535
missing_bases, 'unordered', True)
2536
files.insert_record_stream(missing_entries)
2537
# Now 'merged' is fully inserted (and a commit would succeed).
2538
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2540
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2543
def test_iter_lines_added_or_present_in_keys(self):
2544
# test that we get at least an equalset of the lines added by
2545
# versions in the store.
2546
# the ordering here is to make a tree so that dumb searches have
2547
# more changes to muck up.
2549
class InstrumentedProgress(progress.ProgressTask):
2552
progress.ProgressTask.__init__(self)
2555
def update(self, msg=None, current=None, total=None):
2556
self.updates.append((msg, current, total))
2558
files = self.get_versionedfiles()
2559
# add a base to get included
2560
files.add_lines(self.get_simple_key(b'base'), (), [b'base\n'])
2561
# add a ancestor to be included on one side
2562
files.add_lines(self.get_simple_key(b'lancestor'), (), [b'lancestor\n'])
2563
# add a ancestor to be included on the other side
2564
files.add_lines(self.get_simple_key(b'rancestor'),
2565
self.get_parents([self.get_simple_key(b'base')]), [b'rancestor\n'])
2566
# add a child of rancestor with no eofile-nl
2567
files.add_lines(self.get_simple_key(b'child'),
2568
self.get_parents([self.get_simple_key(b'rancestor')]),
2569
[b'base\n', b'child\n'])
2570
# add a child of lancestor and base to join the two roots
2571
files.add_lines(self.get_simple_key(b'otherchild'),
2572
self.get_parents([self.get_simple_key(b'lancestor'),
2573
self.get_simple_key(b'base')]),
2574
[b'base\n', b'lancestor\n', b'otherchild\n'])
2575
def iter_with_keys(keys, expected):
2576
# now we need to see what lines are returned, and how often.
2578
progress = InstrumentedProgress()
2579
# iterate over the lines
2580
for line in files.iter_lines_added_or_present_in_keys(keys,
2582
lines.setdefault(line, 0)
2584
if []!= progress.updates:
2585
self.assertEqual(expected, progress.updates)
2587
lines = iter_with_keys(
2588
[self.get_simple_key(b'child'), self.get_simple_key(b'otherchild')],
2589
[('Walking content', 0, 2),
2590
('Walking content', 1, 2),
2591
('Walking content', 2, 2)])
2592
# we must see child and otherchild
2593
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2595
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2596
# we dont care if we got more than that.
2599
lines = iter_with_keys(files.keys(),
2600
[('Walking content', 0, 5),
2601
('Walking content', 1, 5),
2602
('Walking content', 2, 5),
2603
('Walking content', 3, 5),
2604
('Walking content', 4, 5),
2605
('Walking content', 5, 5)])
2606
# all lines must be seen at least once
2607
self.assertTrue(lines[(b'base\n', self.get_simple_key(b'base'))] > 0)
2609
lines[(b'lancestor\n', self.get_simple_key(b'lancestor'))] > 0)
2611
lines[(b'rancestor\n', self.get_simple_key(b'rancestor'))] > 0)
2612
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2614
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2616
def test_make_mpdiffs(self):
2617
from breezy import multiparent
2618
files = self.get_versionedfiles('source')
2619
# add texts that should trip the knit maximum delta chain threshold
2620
# as well as doing parallel chains of data in knits.
2621
# this is done by two chains of 25 insertions
2622
files.add_lines(self.get_simple_key(b'base'), [], [b'line\n'])
2623
files.add_lines(self.get_simple_key(b'noeol'),
2624
self.get_parents([self.get_simple_key(b'base')]), [b'line'])
2625
# detailed eol tests:
2626
# shared last line with parent no-eol
2627
files.add_lines(self.get_simple_key(b'noeolsecond'),
2628
self.get_parents([self.get_simple_key(b'noeol')]),
2629
[b'line\n', b'line'])
2630
# differing last line with parent, both no-eol
2631
files.add_lines(self.get_simple_key(b'noeolnotshared'),
2632
self.get_parents([self.get_simple_key(b'noeolsecond')]),
2633
[b'line\n', b'phone'])
2634
# add eol following a noneol parent, change content
2635
files.add_lines(self.get_simple_key(b'eol'),
2636
self.get_parents([self.get_simple_key(b'noeol')]), [b'phone\n'])
2637
# add eol following a noneol parent, no change content
2638
files.add_lines(self.get_simple_key(b'eolline'),
2639
self.get_parents([self.get_simple_key(b'noeol')]), [b'line\n'])
2640
# noeol with no parents:
2641
files.add_lines(self.get_simple_key(b'noeolbase'), [], [b'line'])
2642
# noeol preceeding its leftmost parent in the output:
2643
# this is done by making it a merge of two parents with no common
2644
# anestry: noeolbase and noeol with the
2645
# later-inserted parent the leftmost.
2646
files.add_lines(self.get_simple_key(b'eolbeforefirstparent'),
2647
self.get_parents([self.get_simple_key(b'noeolbase'),
2648
self.get_simple_key(b'noeol')]),
2650
# two identical eol texts
2651
files.add_lines(self.get_simple_key(b'noeoldup'),
2652
self.get_parents([self.get_simple_key(b'noeol')]), [b'line'])
2653
next_parent = self.get_simple_key(b'base')
2654
text_name = b'chain1-'
2656
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2657
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2658
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2659
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2660
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
2661
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
2662
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2663
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2664
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
2665
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2666
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2667
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2668
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
2669
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2670
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
2671
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2672
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
2673
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2674
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
2675
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
2676
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2677
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2678
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2679
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2680
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2681
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2683
for depth in range(26):
2684
new_version = self.get_simple_key(text_name + b'%d' % depth)
2685
text = text + [b'line\n']
2686
files.add_lines(new_version, self.get_parents([next_parent]), text)
2687
next_parent = new_version
2688
next_parent = self.get_simple_key(b'base')
2689
text_name = b'chain2-'
2691
for depth in range(26):
2692
new_version = self.get_simple_key(text_name + b'%d' % depth)
2693
text = text + [b'line\n']
2694
files.add_lines(new_version, self.get_parents([next_parent]), text)
2695
next_parent = new_version
2696
target = self.get_versionedfiles('target')
2697
for key in multiparent.topo_iter_keys(files, files.keys()):
2698
mpdiff = files.make_mpdiffs([key])[0]
2699
parents = files.get_parent_map([key])[key] or []
2701
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2702
self.assertEqualDiff(
2703
next(files.get_record_stream([key], 'unordered',
2704
True)).get_bytes_as('fulltext'),
2705
next(target.get_record_stream([key], 'unordered',
2706
True)).get_bytes_as('fulltext')
2709
def test_keys(self):
2710
# While use is discouraged, versions() is still needed by aspects of
2712
files = self.get_versionedfiles()
2713
self.assertEqual(set(), set(files.keys()))
2714
if self.key_length == 1:
2717
key = (b'foo', b'bar',)
2718
files.add_lines(key, (), [])
2719
self.assertEqual({key}, set(files.keys()))
2722
class VirtualVersionedFilesTests(TestCase):
2723
"""Basic tests for the VirtualVersionedFiles implementations."""
2725
def _get_parent_map(self, keys):
2728
if k in self._parent_map:
2729
ret[k] = self._parent_map[k]
2733
super(VirtualVersionedFilesTests, self).setUp()
2735
self._parent_map = {}
2736
self.texts = VirtualVersionedFiles(self._get_parent_map,
2739
def test_add_lines(self):
2740
self.assertRaises(NotImplementedError,
2741
self.texts.add_lines, b"foo", [], [])
2743
def test_add_mpdiffs(self):
2744
self.assertRaises(NotImplementedError,
2745
self.texts.add_mpdiffs, [])
2747
def test_check_noerrors(self):
2750
def test_insert_record_stream(self):
2751
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2754
def test_get_sha1s_nonexistent(self):
2755
self.assertEqual({}, self.texts.get_sha1s([(b"NONEXISTENT",)]))
2757
def test_get_sha1s(self):
2758
self._lines[b"key"] = [b"dataline1", b"dataline2"]
2759
self.assertEqual({(b"key",): osutils.sha_strings(self._lines[b"key"])},
2760
self.texts.get_sha1s([(b"key",)]))
2762
def test_get_parent_map(self):
2763
self._parent_map = {b"G": (b"A", b"B")}
2764
self.assertEqual({(b"G",): ((b"A",), (b"B",))},
2765
self.texts.get_parent_map([(b"G",), (b"L",)]))
2767
def test_get_record_stream(self):
2768
self._lines[b"A"] = [b"FOO", b"BAR"]
2769
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2771
self.assertEqual("chunked", record.storage_kind)
2772
self.assertEqual(b"FOOBAR", record.get_bytes_as("fulltext"))
2773
self.assertEqual([b"FOO", b"BAR"], record.get_bytes_as("chunked"))
2775
def test_get_record_stream_absent(self):
2776
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2778
self.assertEqual("absent", record.storage_kind)
2780
def test_iter_lines_added_or_present_in_keys(self):
2781
self._lines[b"A"] = [b"FOO", b"BAR"]
2782
self._lines[b"B"] = [b"HEY"]
2783
self._lines[b"C"] = [b"Alberta"]
2784
it = self.texts.iter_lines_added_or_present_in_keys([(b"A",), (b"B",)])
2785
self.assertEqual(sorted([(b"FOO", b"A"), (b"BAR", b"A"), (b"HEY", b"B")]),
2789
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2791
def get_ordering_vf(self, key_priority):
2792
builder = self.make_branch_builder('test')
2793
builder.start_series()
2794
builder.build_snapshot(None, [
2795
('add', ('', b'TREE_ROOT', 'directory', None))],
2797
builder.build_snapshot([b'A'], [], revision_id=b'B')
2798
builder.build_snapshot([b'B'], [], revision_id=b'C')
2799
builder.build_snapshot([b'C'], [], revision_id=b'D')
2800
builder.finish_series()
2801
b = builder.get_branch()
2803
self.addCleanup(b.unlock)
2804
vf = b.repository.inventories
2805
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2807
def test_get_empty(self):
2808
vf = self.get_ordering_vf({})
2809
self.assertEqual([], vf.calls)
2811
def test_get_record_stream_topological(self):
2812
vf = self.get_ordering_vf({(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2813
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2814
keys = [r.key for r in vf.get_record_stream(request_keys,
2815
'topological', False)]
2816
# We should have gotten the keys in topological order
2817
self.assertEqual([(b'A',), (b'B',), (b'C',), (b'D',)], keys)
2818
# And recorded that the request was made
2819
self.assertEqual([('get_record_stream', request_keys, 'topological',
2822
def test_get_record_stream_ordered(self):
2823
vf = self.get_ordering_vf({(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2824
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2825
keys = [r.key for r in vf.get_record_stream(request_keys,
2826
'unordered', False)]
2827
# They should be returned based on their priority
2828
self.assertEqual([(b'D',), (b'B',), (b'A',), (b'C',)], keys)
2829
# And the request recorded
2830
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2833
def test_get_record_stream_implicit_order(self):
2834
vf = self.get_ordering_vf({(b'B',): 2, (b'D',): 1})
2835
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2836
keys = [r.key for r in vf.get_record_stream(request_keys,
2837
'unordered', False)]
2838
# A and C are not in the map, so they get sorted to the front. A comes
2839
# before C alphabetically, so it comes back first
2840
self.assertEqual([(b'A',), (b'C',), (b'D',), (b'B',)], keys)
2841
# And the request recorded
2842
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
breezy/tests/per_versionedfile.py
