1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
24
from gzip import GzipFile
39
from ..errors import (
41
RevisionAlreadyPresent,
43
from ..bzr.knit import (
48
from ..sixish import (
54
TestCaseWithMemoryTransport,
58
from .http_utils import TestCaseWithWebserver
59
from ..transport.memory import MemoryTransport
60
from ..bzr import versionedfile as versionedfile
61
from ..bzr.versionedfile import (
63
HashEscapedPrefixMapper,
65
VirtualVersionedFiles,
66
make_versioned_files_factory,
68
from ..bzr.weave import (
72
from ..bzr.weavefile import write_weave
73
from .scenarios import load_tests_apply_scenarios
76
load_tests = load_tests_apply_scenarios
79
def get_diamond_vf(f, trailing_eol=True, left_only=False):
80
"""Get a diamond graph to exercise deltas and merges.
82
:param trailing_eol: If True end the last line with \n.
86
b'base': ((b'origin',),),
87
b'left': ((b'base',),),
88
b'right': ((b'base',),),
89
b'merged': ((b'left',), (b'right',)),
91
# insert a diamond graph to exercise deltas and merges.
96
f.add_lines(b'origin', [], [b'origin' + last_char])
97
f.add_lines(b'base', [b'origin'], [b'base' + last_char])
98
f.add_lines(b'left', [b'base'], [b'base\n', b'left' + last_char])
100
f.add_lines(b'right', [b'base'],
101
[b'base\n', b'right' + last_char])
102
f.add_lines(b'merged', [b'left', b'right'],
103
[b'base\n', b'left\n', b'right\n', b'merged' + last_char])
107
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
108
nograph=False, nokeys=False):
109
"""Get a diamond graph to exercise deltas and merges.
111
This creates a 5-node graph in files. If files supports 2-length keys two
112
graphs are made to exercise the support for multiple ids.
114
:param trailing_eol: If True end the last line with \n.
115
:param key_length: The length of keys in files. Currently supports length 1
117
:param left_only: If True do not add the right and merged nodes.
118
:param nograph: If True, do not provide parents to the add_lines calls;
119
this is useful for tests that need inserted data but have graphless
121
:param nokeys: If True, pass None is as the key for all insertions.
122
Currently implies nograph.
123
:return: The results of the add_lines calls.
130
prefixes = [(b'FileA',), (b'FileB',)]
131
# insert a diamond graph to exercise deltas and merges.
138
def get_parents(suffix_list):
142
result = [prefix + suffix for suffix in suffix_list]
150
# we loop over each key because that spreads the inserts across prefixes,
151
# which is how commit operates.
152
for prefix in prefixes:
153
result.append(files.add_lines(prefix + get_key(b'origin'), (),
154
[b'origin' + last_char]))
155
for prefix in prefixes:
156
result.append(files.add_lines(prefix + get_key(b'base'),
157
get_parents([(b'origin',)]), [b'base' + last_char]))
158
for prefix in prefixes:
159
result.append(files.add_lines(prefix + get_key(b'left'),
160
get_parents([(b'base',)]),
161
[b'base\n', b'left' + last_char]))
163
for prefix in prefixes:
164
result.append(files.add_lines(prefix + get_key(b'right'),
165
get_parents([(b'base',)]),
166
[b'base\n', b'right' + last_char]))
167
for prefix in prefixes:
168
result.append(files.add_lines(prefix + get_key(b'merged'),
170
[(b'left',), (b'right',)]),
171
[b'base\n', b'left\n', b'right\n', b'merged' + last_char]))
175
class VersionedFileTestMixIn(object):
176
"""A mixin test class for testing VersionedFiles.
178
This is not an adaptor-style test at this point because
179
theres no dynamic substitution of versioned file implementations,
180
they are strictly controlled by their owning repositories.
183
def get_transaction(self):
184
if not hasattr(self, '_transaction'):
185
self._transaction = None
186
return self._transaction
190
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
191
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
194
versions = f.versions()
195
self.assertTrue(b'r0' in versions)
196
self.assertTrue(b'r1' in versions)
197
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
198
self.assertEqual(f.get_text(b'r0'), b'a\nb\n')
199
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
200
self.assertEqual(2, len(f))
201
self.assertEqual(2, f.num_versions())
203
self.assertRaises(RevisionNotPresent,
204
f.add_lines, b'r2', [b'foo'], [])
205
self.assertRaises(RevisionAlreadyPresent,
206
f.add_lines, b'r1', [], [])
208
# this checks that reopen with create=True does not break anything.
209
f = self.reopen_file(create=True)
212
def test_adds_with_parent_texts(self):
215
_, _, parent_texts[b'r0'] = f.add_lines(b'r0', [], [b'a\n', b'b\n'])
217
_, _, parent_texts[b'r1'] = f.add_lines_with_ghosts(b'r1',
218
[b'r0', b'ghost'], [b'b\n', b'c\n'], parent_texts=parent_texts)
219
except NotImplementedError:
220
# if the format doesn't support ghosts, just add normally.
221
_, _, parent_texts[b'r1'] = f.add_lines(b'r1',
222
[b'r0'], [b'b\n', b'c\n'], parent_texts=parent_texts)
223
f.add_lines(b'r2', [b'r1'], [b'c\n', b'd\n'],
224
parent_texts=parent_texts)
225
self.assertNotEqual(None, parent_texts[b'r0'])
226
self.assertNotEqual(None, parent_texts[b'r1'])
229
versions = f.versions()
230
self.assertTrue(b'r0' in versions)
231
self.assertTrue(b'r1' in versions)
232
self.assertTrue(b'r2' in versions)
233
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
234
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
235
self.assertEqual(f.get_lines(b'r2'), [b'c\n', b'd\n'])
236
self.assertEqual(3, f.num_versions())
237
origins = f.annotate(b'r1')
238
self.assertEqual(origins[0][0], b'r0')
239
self.assertEqual(origins[1][0], b'r1')
240
origins = f.annotate(b'r2')
241
self.assertEqual(origins[0][0], b'r1')
242
self.assertEqual(origins[1][0], b'r2')
245
f = self.reopen_file()
248
def test_add_unicode_content(self):
249
# unicode content is not permitted in versioned files.
250
# versioned files version sequences of bytes only.
252
self.assertRaises(errors.BzrBadParameterUnicode,
253
vf.add_lines, b'a', [], [b'a\n', u'b\n', b'c\n'])
255
(errors.BzrBadParameterUnicode, NotImplementedError),
256
vf.add_lines_with_ghosts, b'a', [], [b'a\n', u'b\n', b'c\n'])
258
def test_add_follows_left_matching_blocks(self):
259
"""If we change left_matching_blocks, delta changes
261
Note: There are multiple correct deltas in this case, because
262
we start with 1 "a" and we get 3.
265
if isinstance(vf, WeaveFile):
266
raise TestSkipped("WeaveFile ignores left_matching_blocks")
267
vf.add_lines(b'1', [], [b'a\n'])
268
vf.add_lines(b'2', [b'1'], [b'a\n', b'a\n', b'a\n'],
269
left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
270
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'2'))
271
vf.add_lines(b'3', [b'1'], [b'a\n', b'a\n', b'a\n'],
272
left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
273
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'3'))
275
def test_inline_newline_throws(self):
276
# \r characters are not permitted in lines being added
278
self.assertRaises(errors.BzrBadParameterContainsNewline,
279
vf.add_lines, b'a', [], [b'a\n\n'])
281
(errors.BzrBadParameterContainsNewline, NotImplementedError),
282
vf.add_lines_with_ghosts, b'a', [], [b'a\n\n'])
283
# but inline CR's are allowed
284
vf.add_lines(b'a', [], [b'a\r\n'])
286
vf.add_lines_with_ghosts(b'b', [], [b'a\r\n'])
287
except NotImplementedError:
290
def test_add_reserved(self):
292
self.assertRaises(errors.ReservedId,
293
vf.add_lines, b'a:', [], [b'a\n', b'b\n', b'c\n'])
295
def test_add_lines_nostoresha(self):
296
"""When nostore_sha is supplied using old content raises."""
298
empty_text = (b'a', [])
299
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
300
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
302
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
303
sha, _, _ = vf.add_lines(version, [], lines)
305
# we now have a copy of all the lines in the vf.
306
for sha, (version, lines) in zip(
307
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
308
self.assertRaises(errors.ExistingContent,
309
vf.add_lines, version + b"2", [], lines,
311
# and no new version should have been added.
312
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
315
def test_add_lines_with_ghosts_nostoresha(self):
316
"""When nostore_sha is supplied using old content raises."""
318
empty_text = (b'a', [])
319
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
320
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
322
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
323
sha, _, _ = vf.add_lines(version, [], lines)
325
# we now have a copy of all the lines in the vf.
326
# is the test applicable to this vf implementation?
328
vf.add_lines_with_ghosts(b'd', [], [])
329
except NotImplementedError:
330
raise TestSkipped("add_lines_with_ghosts is optional")
331
for sha, (version, lines) in zip(
332
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
333
self.assertRaises(errors.ExistingContent,
334
vf.add_lines_with_ghosts, version + b"2", [], lines,
336
# and no new version should have been added.
337
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
340
def test_add_lines_return_value(self):
341
# add_lines should return the sha1 and the text size.
343
empty_text = (b'a', [])
344
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
345
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
346
# check results for the three cases:
347
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
348
# the first two elements are the same for all versioned files:
349
# - the digest and the size of the text. For some versioned files
350
# additional data is returned in additional tuple elements.
351
result = vf.add_lines(version, [], lines)
352
self.assertEqual(3, len(result))
353
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
355
# parents should not affect the result:
356
lines = sample_text_nl[1]
357
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
358
vf.add_lines(b'd', [b'b', b'c'], lines)[0:2])
360
def test_get_reserved(self):
362
self.assertRaises(errors.ReservedId, vf.get_texts, [b'b:'])
363
self.assertRaises(errors.ReservedId, vf.get_lines, b'b:')
364
self.assertRaises(errors.ReservedId, vf.get_text, b'b:')
366
def test_add_unchanged_last_line_noeol_snapshot(self):
367
"""Add a text with an unchanged last line with no eol should work."""
368
# Test adding this in a number of chain lengths; because the interface
369
# for VersionedFile does not allow forcing a specific chain length, we
370
# just use a small base to get the first snapshot, then a much longer
371
# first line for the next add (which will make the third add snapshot)
372
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
373
# as a capped delta length, but ideally we would have some way of
374
# tuning the test to the store (e.g. keep going until a snapshot
376
for length in range(20):
378
vf = self.get_file('case-%d' % length)
381
for step in range(length):
382
version = prefix % step
383
lines = ([b'prelude \n'] * step) + [b'line']
384
vf.add_lines(version, parents, lines)
385
version_lines[version] = lines
387
vf.add_lines(b'no-eol', parents, [b'line'])
388
vf.get_texts(version_lines.keys())
389
self.assertEqualDiff(b'line', vf.get_text(b'no-eol'))
391
def test_get_texts_eol_variation(self):
392
# similar to the failure in <http://bugs.launchpad.net/234748>
394
sample_text_nl = [b"line\n"]
395
sample_text_no_nl = [b"line"]
402
lines = sample_text_nl
404
lines = sample_text_no_nl
405
# left_matching blocks is an internal api; it operates on the
406
# *internal* representation for a knit, which is with *all* lines
407
# being normalised to end with \n - even the final line in a no_nl
408
# file. Using it here ensures that a broken internal implementation
409
# (which is what this test tests) will generate a correct line
410
# delta (which is to say, an empty delta).
411
vf.add_lines(version, parents, lines,
412
left_matching_blocks=[(0, 0, 1)])
414
versions.append(version)
415
version_lines[version] = lines
417
vf.get_texts(versions)
418
vf.get_texts(reversed(versions))
420
def test_add_lines_with_matching_blocks_noeol_last_line(self):
421
"""Add a text with an unchanged last line with no eol should work."""
422
from breezy import multiparent
423
# Hand verified sha1 of the text we're adding.
424
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
425
# Create a mpdiff which adds a new line before the trailing line, and
426
# reuse the last line unaltered (which can cause annotation reuse).
427
# Test adding this in two situations:
428
# On top of a new insertion
429
vf = self.get_file('fulltext')
430
vf.add_lines(b'noeol', [], [b'line'])
431
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
432
left_matching_blocks=[(0, 1, 1)])
433
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
435
vf = self.get_file('delta')
436
vf.add_lines(b'base', [], [b'line'])
437
vf.add_lines(b'noeol', [b'base'], [b'prelude\n', b'line'])
438
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
439
left_matching_blocks=[(1, 1, 1)])
440
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
442
def test_make_mpdiffs(self):
443
from breezy import multiparent
444
vf = self.get_file('foo')
445
sha1s = self._setup_for_deltas(vf)
446
new_vf = self.get_file('bar')
447
for version in multiparent.topo_iter(vf):
448
mpdiff = vf.make_mpdiffs([version])[0]
449
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
450
vf.get_sha1s([version])[version], mpdiff)])
451
self.assertEqualDiff(vf.get_text(version),
452
new_vf.get_text(version))
454
def test_make_mpdiffs_with_ghosts(self):
455
vf = self.get_file('foo')
457
vf.add_lines_with_ghosts(b'text', [b'ghost'], [b'line\n'])
458
except NotImplementedError:
459
# old Weave formats do not allow ghosts
461
self.assertRaises(errors.RevisionNotPresent,
462
vf.make_mpdiffs, [b'ghost'])
464
def _setup_for_deltas(self, f):
465
self.assertFalse(f.has_version('base'))
466
# add texts that should trip the knit maximum delta chain threshold
467
# as well as doing parallel chains of data in knits.
468
# this is done by two chains of 25 insertions
469
f.add_lines(b'base', [], [b'line\n'])
470
f.add_lines(b'noeol', [b'base'], [b'line'])
471
# detailed eol tests:
472
# shared last line with parent no-eol
473
f.add_lines(b'noeolsecond', [b'noeol'], [b'line\n', b'line'])
474
# differing last line with parent, both no-eol
475
f.add_lines(b'noeolnotshared', [b'noeolsecond'], [b'line\n', b'phone'])
476
# add eol following a noneol parent, change content
477
f.add_lines(b'eol', [b'noeol'], [b'phone\n'])
478
# add eol following a noneol parent, no change content
479
f.add_lines(b'eolline', [b'noeol'], [b'line\n'])
480
# noeol with no parents:
481
f.add_lines(b'noeolbase', [], [b'line'])
482
# noeol preceeding its leftmost parent in the output:
483
# this is done by making it a merge of two parents with no common
484
# anestry: noeolbase and noeol with the
485
# later-inserted parent the leftmost.
486
f.add_lines(b'eolbeforefirstparent', [
487
b'noeolbase', b'noeol'], [b'line'])
488
# two identical eol texts
489
f.add_lines(b'noeoldup', [b'noeol'], [b'line'])
490
next_parent = b'base'
491
text_name = b'chain1-'
493
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
494
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
495
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
496
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
497
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
498
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
499
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
500
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
501
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
502
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
503
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
504
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
505
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
506
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
507
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
508
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
509
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
510
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
511
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
512
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
513
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
514
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
515
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
516
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
517
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
518
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
520
for depth in range(26):
521
new_version = text_name + b'%d' % depth
522
text = text + [b'line\n']
523
f.add_lines(new_version, [next_parent], text)
524
next_parent = new_version
525
next_parent = b'base'
526
text_name = b'chain2-'
528
for depth in range(26):
529
new_version = text_name + b'%d' % depth
530
text = text + [b'line\n']
531
f.add_lines(new_version, [next_parent], text)
532
next_parent = new_version
535
def test_ancestry(self):
537
self.assertEqual([], f.get_ancestry([]))
538
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
539
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
540
f.add_lines(b'r2', [b'r0'], [b'b\n', b'c\n'])
541
f.add_lines(b'r3', [b'r2'], [b'b\n', b'c\n'])
542
f.add_lines(b'rM', [b'r1', b'r2'], [b'b\n', b'c\n'])
543
self.assertEqual([], f.get_ancestry([]))
544
versions = f.get_ancestry([b'rM'])
545
# there are some possibilities:
549
# so we check indexes
550
r0 = versions.index(b'r0')
551
r1 = versions.index(b'r1')
552
r2 = versions.index(b'r2')
553
self.assertFalse(b'r3' in versions)
554
rM = versions.index(b'rM')
555
self.assertTrue(r0 < r1)
556
self.assertTrue(r0 < r2)
557
self.assertTrue(r1 < rM)
558
self.assertTrue(r2 < rM)
560
self.assertRaises(RevisionNotPresent,
561
f.get_ancestry, [b'rM', b'rX'])
563
self.assertEqual(set(f.get_ancestry(b'rM')),
564
set(f.get_ancestry(b'rM', topo_sorted=False)))
566
def test_mutate_after_finish(self):
567
self._transaction = 'before'
569
self._transaction = 'after'
570
self.assertRaises(errors.OutSideTransaction, f.add_lines, b'', [], [])
571
self.assertRaises(errors.OutSideTransaction,
572
f.add_lines_with_ghosts, b'', [], [])
574
def test_copy_to(self):
576
f.add_lines(b'0', [], [b'a\n'])
577
t = MemoryTransport()
579
for suffix in self.get_factory().get_suffixes():
580
self.assertTrue(t.has('foo' + suffix))
582
def test_get_suffixes(self):
584
# and should be a list
585
self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
587
def test_get_parent_map(self):
589
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
591
{b'r0': ()}, f.get_parent_map([b'r0']))
592
f.add_lines(b'r1', [b'r0'], [b'a\n', b'b\n'])
594
{b'r1': (b'r0',)}, f.get_parent_map([b'r1']))
598
f.get_parent_map([b'r0', b'r1']))
599
f.add_lines(b'r2', [], [b'a\n', b'b\n'])
600
f.add_lines(b'r3', [], [b'a\n', b'b\n'])
601
f.add_lines(b'm', [b'r0', b'r1', b'r2', b'r3'], [b'a\n', b'b\n'])
603
{b'm': (b'r0', b'r1', b'r2', b'r3')}, f.get_parent_map([b'm']))
604
self.assertEqual({}, f.get_parent_map(b'y'))
608
f.get_parent_map([b'r0', b'y', b'r1']))
610
def test_annotate(self):
612
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
613
f.add_lines(b'r1', [b'r0'], [b'c\n', b'b\n'])
614
origins = f.annotate(b'r1')
615
self.assertEqual(origins[0][0], b'r1')
616
self.assertEqual(origins[1][0], b'r0')
618
self.assertRaises(RevisionNotPresent,
621
def test_detection(self):
622
# Test weaves detect corruption.
624
# Weaves contain a checksum of their texts.
625
# When a text is extracted, this checksum should be
628
w = self.get_file_corrupted_text()
630
self.assertEqual(b'hello\n', w.get_text(b'v1'))
631
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
632
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
633
self.assertRaises(WeaveInvalidChecksum, w.check)
635
w = self.get_file_corrupted_checksum()
637
self.assertEqual(b'hello\n', w.get_text(b'v1'))
638
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
639
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
640
self.assertRaises(WeaveInvalidChecksum, w.check)
642
def get_file_corrupted_text(self):
643
"""Return a versioned file with corrupt text but valid metadata."""
644
raise NotImplementedError(self.get_file_corrupted_text)
646
def reopen_file(self, name='foo'):
647
"""Open the versioned file from disk again."""
648
raise NotImplementedError(self.reopen_file)
650
def test_iter_lines_added_or_present_in_versions(self):
651
# test that we get at least an equalset of the lines added by
652
# versions in the weave
653
# the ordering here is to make a tree so that dumb searches have
654
# more changes to muck up.
656
class InstrumentedProgress(progress.ProgressTask):
659
progress.ProgressTask.__init__(self)
662
def update(self, msg=None, current=None, total=None):
663
self.updates.append((msg, current, total))
666
# add a base to get included
667
vf.add_lines(b'base', [], [b'base\n'])
668
# add a ancestor to be included on one side
669
vf.add_lines(b'lancestor', [], [b'lancestor\n'])
670
# add a ancestor to be included on the other side
671
vf.add_lines(b'rancestor', [b'base'], [b'rancestor\n'])
672
# add a child of rancestor with no eofile-nl
673
vf.add_lines(b'child', [b'rancestor'], [b'base\n', b'child\n'])
674
# add a child of lancestor and base to join the two roots
675
vf.add_lines(b'otherchild',
676
[b'lancestor', b'base'],
677
[b'base\n', b'lancestor\n', b'otherchild\n'])
679
def iter_with_versions(versions, expected):
680
# now we need to see what lines are returned, and how often.
682
progress = InstrumentedProgress()
683
# iterate over the lines
684
for line in vf.iter_lines_added_or_present_in_versions(versions,
686
lines.setdefault(line, 0)
688
if [] != progress.updates:
689
self.assertEqual(expected, progress.updates)
691
lines = iter_with_versions([b'child', b'otherchild'],
692
[('Walking content', 0, 2),
693
('Walking content', 1, 2),
694
('Walking content', 2, 2)])
695
# we must see child and otherchild
696
self.assertTrue(lines[(b'child\n', b'child')] > 0)
697
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
698
# we dont care if we got more than that.
701
lines = iter_with_versions(None, [('Walking content', 0, 5),
702
('Walking content', 1, 5),
703
('Walking content', 2, 5),
704
('Walking content', 3, 5),
705
('Walking content', 4, 5),
706
('Walking content', 5, 5)])
707
# all lines must be seen at least once
708
self.assertTrue(lines[(b'base\n', b'base')] > 0)
709
self.assertTrue(lines[(b'lancestor\n', b'lancestor')] > 0)
710
self.assertTrue(lines[(b'rancestor\n', b'rancestor')] > 0)
711
self.assertTrue(lines[(b'child\n', b'child')] > 0)
712
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
714
def test_add_lines_with_ghosts(self):
715
# some versioned file formats allow lines to be added with parent
716
# information that is > than that in the format. Formats that do
717
# not support this need to raise NotImplementedError on the
718
# add_lines_with_ghosts api.
720
# add a revision with ghost parents
721
# The preferred form is utf8, but we should translate when needed
722
parent_id_unicode = u'b\xbfse'
723
parent_id_utf8 = parent_id_unicode.encode('utf8')
725
vf.add_lines_with_ghosts(b'notbxbfse', [parent_id_utf8], [])
726
except NotImplementedError:
727
# check the other ghost apis are also not implemented
728
self.assertRaises(NotImplementedError,
729
vf.get_ancestry_with_ghosts, [b'foo'])
730
self.assertRaises(NotImplementedError,
731
vf.get_parents_with_ghosts, b'foo')
733
vf = self.reopen_file()
734
# test key graph related apis: getncestry, _graph, get_parents
736
# - these are ghost unaware and must not be reflect ghosts
737
self.assertEqual([b'notbxbfse'], vf.get_ancestry(b'notbxbfse'))
738
self.assertFalse(vf.has_version(parent_id_utf8))
739
# we have _with_ghost apis to give us ghost information.
740
self.assertEqual([parent_id_utf8, b'notbxbfse'],
741
vf.get_ancestry_with_ghosts([b'notbxbfse']))
742
self.assertEqual([parent_id_utf8],
743
vf.get_parents_with_ghosts(b'notbxbfse'))
744
# if we add something that is a ghost of another, it should correct the
745
# results of the prior apis
746
vf.add_lines(parent_id_utf8, [], [])
747
self.assertEqual([parent_id_utf8, b'notbxbfse'],
748
vf.get_ancestry([b'notbxbfse']))
749
self.assertEqual({b'notbxbfse': (parent_id_utf8,)},
750
vf.get_parent_map([b'notbxbfse']))
751
self.assertTrue(vf.has_version(parent_id_utf8))
752
# we have _with_ghost apis to give us ghost information.
753
self.assertEqual([parent_id_utf8, b'notbxbfse'],
754
vf.get_ancestry_with_ghosts([b'notbxbfse']))
755
self.assertEqual([parent_id_utf8],
756
vf.get_parents_with_ghosts(b'notbxbfse'))
758
def test_add_lines_with_ghosts_after_normal_revs(self):
759
# some versioned file formats allow lines to be added with parent
760
# information that is > than that in the format. Formats that do
761
# not support this need to raise NotImplementedError on the
762
# add_lines_with_ghosts api.
764
# probe for ghost support
766
vf.add_lines_with_ghosts(b'base', [], [b'line\n', b'line_b\n'])
767
except NotImplementedError:
769
vf.add_lines_with_ghosts(b'references_ghost',
770
[b'base', b'a_ghost'],
771
[b'line\n', b'line_b\n', b'line_c\n'])
772
origins = vf.annotate(b'references_ghost')
773
self.assertEqual((b'base', b'line\n'), origins[0])
774
self.assertEqual((b'base', b'line_b\n'), origins[1])
775
self.assertEqual((b'references_ghost', b'line_c\n'), origins[2])
777
def test_readonly_mode(self):
778
t = self.get_transport()
779
factory = self.get_factory()
780
vf = factory('id', t, 0o777, create=True, access_mode='w')
781
vf = factory('id', t, access_mode='r')
782
self.assertRaises(errors.ReadOnlyError, vf.add_lines, b'base', [], [])
783
self.assertRaises(errors.ReadOnlyError,
784
vf.add_lines_with_ghosts,
789
def test_get_sha1s(self):
790
# check the sha1 data is available
793
vf.add_lines(b'a', [], [b'a\n'])
794
# the same file, different metadata
795
vf.add_lines(b'b', [b'a'], [b'a\n'])
796
# a file differing only in last newline.
797
vf.add_lines(b'c', [], [b'a'])
799
b'a': b'3f786850e387550fdab836ed7e6dc881de23001b',
800
b'c': b'86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
801
b'b': b'3f786850e387550fdab836ed7e6dc881de23001b',
803
vf.get_sha1s([b'a', b'c', b'b']))
806
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
808
def get_file(self, name='foo'):
809
return WeaveFile(name, self.get_transport(),
811
get_scope=self.get_transaction)
813
def get_file_corrupted_text(self):
814
w = WeaveFile('foo', self.get_transport(),
816
get_scope=self.get_transaction)
817
w.add_lines(b'v1', [], [b'hello\n'])
818
w.add_lines(b'v2', [b'v1'], [b'hello\n', b'there\n'])
820
# We are going to invasively corrupt the text
821
# Make sure the internals of weave are the same
822
self.assertEqual([(b'{', 0), b'hello\n', (b'}', None), (b'{', 1), b'there\n', (b'}', None)
825
self.assertEqual([b'f572d396fae9206628714fb2ce00f72e94f2258f', b'90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
830
w._weave[4] = b'There\n'
833
def get_file_corrupted_checksum(self):
834
w = self.get_file_corrupted_text()
836
w._weave[4] = b'there\n'
837
self.assertEqual(b'hello\nthere\n', w.get_text(b'v2'))
839
# Invalid checksum, first digit changed
840
w._sha1s[1] = b'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
843
def reopen_file(self, name='foo', create=False):
844
return WeaveFile(name, self.get_transport(),
846
get_scope=self.get_transaction)
848
def test_no_implicit_create(self):
849
self.assertRaises(errors.NoSuchFile,
852
self.get_transport(),
853
get_scope=self.get_transaction)
855
def get_factory(self):
859
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
862
super(TestPlanMergeVersionedFile, self).setUp()
863
mapper = PrefixMapper()
864
factory = make_file_factory(True, mapper)
865
self.vf1 = factory(self.get_transport('root-1'))
866
self.vf2 = factory(self.get_transport('root-2'))
867
self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
868
self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
870
def test_add_lines(self):
871
self.plan_merge_vf.add_lines((b'root', b'a:'), [], [])
872
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
873
(b'root', b'a'), [], [])
874
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
875
(b'root', b'a:'), None, [])
876
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
877
(b'root', b'a:'), [], None)
879
def setup_abcde(self):
880
self.vf1.add_lines((b'root', b'A'), [], [b'a'])
881
self.vf1.add_lines((b'root', b'B'), [(b'root', b'A')], [b'b'])
882
self.vf2.add_lines((b'root', b'C'), [], [b'c'])
883
self.vf2.add_lines((b'root', b'D'), [(b'root', b'C')], [b'd'])
884
self.plan_merge_vf.add_lines((b'root', b'E:'),
885
[(b'root', b'B'), (b'root', b'D')], [b'e'])
887
def test_get_parents(self):
889
self.assertEqual({(b'root', b'B'): ((b'root', b'A'),)},
890
self.plan_merge_vf.get_parent_map([(b'root', b'B')]))
891
self.assertEqual({(b'root', b'D'): ((b'root', b'C'),)},
892
self.plan_merge_vf.get_parent_map([(b'root', b'D')]))
893
self.assertEqual({(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D'))},
894
self.plan_merge_vf.get_parent_map([(b'root', b'E:')]))
896
self.plan_merge_vf.get_parent_map([(b'root', b'F')]))
898
(b'root', b'B'): ((b'root', b'A'),),
899
(b'root', b'D'): ((b'root', b'C'),),
900
(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D')),
902
self.plan_merge_vf.get_parent_map(
903
[(b'root', b'B'), (b'root', b'D'), (b'root', b'E:'), (b'root', b'F')]))
905
def test_get_record_stream(self):
908
def get_record(suffix):
909
return next(self.plan_merge_vf.get_record_stream(
910
[(b'root', suffix)], 'unordered', True))
911
self.assertEqual(b'a', get_record(b'A').get_bytes_as('fulltext'))
912
self.assertEqual(b'c', get_record(b'C').get_bytes_as('fulltext'))
913
self.assertEqual(b'e', get_record(b'E:').get_bytes_as('fulltext'))
914
self.assertEqual('absent', get_record('F').storage_kind)
917
class TestReadonlyHttpMixin(object):
919
def get_transaction(self):
922
def test_readonly_http_works(self):
923
# we should be able to read from http with a versioned file.
925
# try an empty file access
926
readonly_vf = self.get_factory()('foo',
927
transport.get_transport_from_url(self.get_readonly_url('.')))
928
self.assertEqual([], readonly_vf.versions())
930
def test_readonly_http_works_with_feeling(self):
931
# we should be able to read from http with a versioned file.
934
vf.add_lines(b'1', [], [b'a\n'])
935
vf.add_lines(b'2', [b'1'], [b'b\n', b'a\n'])
936
readonly_vf = self.get_factory()('foo',
937
transport.get_transport_from_url(self.get_readonly_url('.')))
938
self.assertEqual([b'1', b'2'], vf.versions())
939
self.assertEqual([b'1', b'2'], readonly_vf.versions())
940
for version in readonly_vf.versions():
941
readonly_vf.get_lines(version)
944
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
947
return WeaveFile('foo', self.get_transport(),
949
get_scope=self.get_transaction)
951
def get_factory(self):
955
class MergeCasesMixin(object):
957
def doMerge(self, base, a, b, mp):
958
from textwrap import dedent
964
w.add_lines(b'text0', [], list(map(addcrlf, base)))
965
w.add_lines(b'text1', [b'text0'], list(map(addcrlf, a)))
966
w.add_lines(b'text2', [b'text0'], list(map(addcrlf, b)))
970
self.log('merge plan:')
971
p = list(w.plan_merge(b'text1', b'text2'))
972
for state, line in p:
974
self.log('%12s | %s' % (state, line[:-1]))
978
mt.writelines(w.weave_merge(p))
980
self.log(mt.getvalue())
982
mp = list(map(addcrlf, mp))
983
self.assertEqual(mt.readlines(), mp)
985
def testOneInsert(self):
991
def testSeparateInserts(self):
992
self.doMerge([b'aaa', b'bbb', b'ccc'],
993
[b'aaa', b'xxx', b'bbb', b'ccc'],
994
[b'aaa', b'bbb', b'yyy', b'ccc'],
995
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
997
def testSameInsert(self):
998
self.doMerge([b'aaa', b'bbb', b'ccc'],
999
[b'aaa', b'xxx', b'bbb', b'ccc'],
1000
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'],
1001
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
1002
overlappedInsertExpected = [b'aaa', b'xxx', b'yyy', b'bbb']
1004
def testOverlappedInsert(self):
1005
self.doMerge([b'aaa', b'bbb'],
1006
[b'aaa', b'xxx', b'yyy', b'bbb'],
1007
[b'aaa', b'xxx', b'bbb'], self.overlappedInsertExpected)
1009
# really it ought to reduce this to
1010
# [b'aaa', b'xxx', b'yyy', b'bbb']
1012
def testClashReplace(self):
1013
self.doMerge([b'aaa'],
1016
[b'<<<<<<< ', b'xxx', b'=======', b'yyy', b'zzz',
1019
def testNonClashInsert1(self):
1020
self.doMerge([b'aaa'],
1023
[b'<<<<<<< ', b'xxx', b'aaa', b'=======', b'yyy', b'zzz',
1026
def testNonClashInsert2(self):
1027
self.doMerge([b'aaa'],
1032
def testDeleteAndModify(self):
1033
"""Clashing delete and modification.
1035
If one side modifies a region and the other deletes it then
1036
there should be a conflict with one side blank.
1039
#######################################
1040
# skippd, not working yet
1043
self.doMerge([b'aaa', b'bbb', b'ccc'],
1044
[b'aaa', b'ddd', b'ccc'],
1046
[b'<<<<<<<< ', b'aaa', b'=======', b'>>>>>>> ', b'ccc'])
1048
def _test_merge_from_strings(self, base, a, b, expected):
1050
w.add_lines(b'text0', [], base.splitlines(True))
1051
w.add_lines(b'text1', [b'text0'], a.splitlines(True))
1052
w.add_lines(b'text2', [b'text0'], b.splitlines(True))
1053
self.log('merge plan:')
1054
p = list(w.plan_merge(b'text1', b'text2'))
1055
for state, line in p:
1057
self.log('%12s | %s' % (state, line[:-1]))
1058
self.log('merge result:')
1059
result_text = b''.join(w.weave_merge(p))
1060
self.log(result_text)
1061
self.assertEqualDiff(result_text, expected)
1063
def test_weave_merge_conflicts(self):
1064
# does weave merge properly handle plans that end with unchanged?
1065
result = b''.join(self.get_file().weave_merge([('new-a', b'hello\n')]))
1066
self.assertEqual(result, b'hello\n')
1068
def test_deletion_extended(self):
1069
"""One side deletes, the other deletes more.
1090
self._test_merge_from_strings(base, a, b, result)
1092
def test_deletion_overlap(self):
1093
"""Delete overlapping regions with no other conflict.
1095
Arguably it'd be better to treat these as agreement, rather than
1096
conflict, but for now conflict is safer.
1124
self._test_merge_from_strings(base, a, b, result)
1126
def test_agreement_deletion(self):
1127
"""Agree to delete some lines, without conflicts."""
1149
self._test_merge_from_strings(base, a, b, result)
1151
def test_sync_on_deletion(self):
1152
"""Specific case of merge where we can synchronize incorrectly.
1154
A previous version of the weave merge concluded that the two versions
1155
agreed on deleting line 2, and this could be a synchronization point.
1156
Line 1 was then considered in isolation, and thought to be deleted on
1159
It's better to consider the whole thing as a disagreement region.
1170
a's replacement line 2
1183
a's replacement line 2
1190
self._test_merge_from_strings(base, a, b, result)
1193
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1195
def get_file(self, name='foo'):
1196
return WeaveFile(name, self.get_transport(),
1199
def log_contents(self, w):
1200
self.log('weave is:')
1202
write_weave(w, tmpf)
1203
self.log(tmpf.getvalue())
1205
overlappedInsertExpected = [b'aaa', b'<<<<<<< ', b'xxx', b'yyy', b'=======',
1206
b'xxx', b'>>>>>>> ', b'bbb']
1209
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1211
def test_select_adaptor(self):
1212
"""Test expected adapters exist."""
1213
# One scenario for each lookup combination we expect to use.
1214
# Each is source_kind, requested_kind, adapter class
1216
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1217
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1218
('knit-annotated-delta-gz', 'knit-delta-gz',
1219
_mod_knit.DeltaAnnotatedToUnannotated),
1220
('knit-annotated-delta-gz', 'fulltext',
1221
_mod_knit.DeltaAnnotatedToFullText),
1222
('knit-annotated-ft-gz', 'knit-ft-gz',
1223
_mod_knit.FTAnnotatedToUnannotated),
1224
('knit-annotated-ft-gz', 'fulltext',
1225
_mod_knit.FTAnnotatedToFullText),
1227
for source, requested, klass in scenarios:
1228
adapter_factory = versionedfile.adapter_registry.get(
1229
(source, requested))
1230
adapter = adapter_factory(None)
1231
self.assertIsInstance(adapter, klass)
1233
def get_knit(self, annotated=True):
1234
mapper = ConstantMapper('knit')
1235
transport = self.get_transport()
1236
return make_file_factory(annotated, mapper)(transport)
1238
def helpGetBytes(self, f, ft_adapter, delta_adapter):
1239
"""Grab the interested adapted texts for tests."""
1240
# origin is a fulltext
1241
entries = f.get_record_stream([(b'origin',)], 'unordered', False)
1242
base = next(entries)
1243
ft_data = ft_adapter.get_bytes(base)
1244
# merged is both a delta and multiple parents.
1245
entries = f.get_record_stream([(b'merged',)], 'unordered', False)
1246
merged = next(entries)
1247
delta_data = delta_adapter.get_bytes(merged)
1248
return ft_data, delta_data
1250
def test_deannotation_noeol(self):
1251
"""Test converting annotated knits to unannotated knits."""
1252
# we need a full text, and a delta
1254
get_diamond_files(f, 1, trailing_eol=False)
1255
ft_data, delta_data = self.helpGetBytes(f,
1256
_mod_knit.FTAnnotatedToUnannotated(
1258
_mod_knit.DeltaAnnotatedToUnannotated(None))
1260
b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1263
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1265
b'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1266
b'1,2,3\nleft\nright\nmerged\nend merged\n',
1267
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1269
def test_deannotation(self):
1270
"""Test converting annotated knits to unannotated knits."""
1271
# we need a full text, and a delta
1273
get_diamond_files(f, 1)
1274
ft_data, delta_data = self.helpGetBytes(f,
1275
_mod_knit.FTAnnotatedToUnannotated(
1277
_mod_knit.DeltaAnnotatedToUnannotated(None))
1279
b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1282
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1284
b'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1285
b'2,2,2\nright\nmerged\nend merged\n',
1286
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1288
def test_annotated_to_fulltext_no_eol(self):
1289
"""Test adapting annotated knits to full texts (for -> weaves)."""
1290
# we need a full text, and a delta
1292
get_diamond_files(f, 1, trailing_eol=False)
1293
# Reconstructing a full text requires a backing versioned file, and it
1294
# must have the base lines requested from it.
1295
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1296
ft_data, delta_data = self.helpGetBytes(f,
1297
_mod_knit.FTAnnotatedToFullText(
1299
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1300
self.assertEqual(b'origin', ft_data)
1301
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1302
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1303
True)], logged_vf.calls)
1305
def test_annotated_to_fulltext(self):
1306
"""Test adapting annotated knits to full texts (for -> weaves)."""
1307
# we need a full text, and a delta
1309
get_diamond_files(f, 1)
1310
# Reconstructing a full text requires a backing versioned file, and it
1311
# must have the base lines requested from it.
1312
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1313
ft_data, delta_data = self.helpGetBytes(f,
1314
_mod_knit.FTAnnotatedToFullText(
1316
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1317
self.assertEqual(b'origin\n', ft_data)
1318
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1319
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1320
True)], logged_vf.calls)
1322
def test_unannotated_to_fulltext(self):
1323
"""Test adapting unannotated knits to full texts.
1325
This is used for -> weaves, and for -> annotated knits.
1327
# we need a full text, and a delta
1328
f = self.get_knit(annotated=False)
1329
get_diamond_files(f, 1)
1330
# Reconstructing a full text requires a backing versioned file, and it
1331
# must have the base lines requested from it.
1332
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1333
ft_data, delta_data = self.helpGetBytes(f,
1334
_mod_knit.FTPlainToFullText(
1336
_mod_knit.DeltaPlainToFullText(logged_vf))
1337
self.assertEqual(b'origin\n', ft_data)
1338
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1339
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1340
True)], logged_vf.calls)
1342
def test_unannotated_to_fulltext_no_eol(self):
1343
"""Test adapting unannotated knits to full texts.
1345
This is used for -> weaves, and for -> annotated knits.
1347
# we need a full text, and a delta
1348
f = self.get_knit(annotated=False)
1349
get_diamond_files(f, 1, trailing_eol=False)
1350
# Reconstructing a full text requires a backing versioned file, and it
1351
# must have the base lines requested from it.
1352
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1353
ft_data, delta_data = self.helpGetBytes(f,
1354
_mod_knit.FTPlainToFullText(
1356
_mod_knit.DeltaPlainToFullText(logged_vf))
1357
self.assertEqual(b'origin', ft_data)
1358
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1359
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1360
True)], logged_vf.calls)
1363
class TestKeyMapper(TestCaseWithMemoryTransport):
1364
"""Tests for various key mapping logic."""
1366
def test_identity_mapper(self):
1367
mapper = versionedfile.ConstantMapper("inventory")
1368
self.assertEqual("inventory", mapper.map((b'foo@ar',)))
1369
self.assertEqual("inventory", mapper.map((b'quux',)))
1371
def test_prefix_mapper(self):
1373
mapper = versionedfile.PrefixMapper()
1374
self.assertEqual("file-id", mapper.map((b"file-id", b"revision-id")))
1375
self.assertEqual("new-id", mapper.map((b"new-id", b"revision-id")))
1376
self.assertEqual((b'file-id',), mapper.unmap("file-id"))
1377
self.assertEqual((b'new-id',), mapper.unmap("new-id"))
1379
def test_hash_prefix_mapper(self):
1380
#format6: hash + plain
1381
mapper = versionedfile.HashPrefixMapper()
1383
"9b/file-id", mapper.map((b"file-id", b"revision-id")))
1384
self.assertEqual("45/new-id", mapper.map((b"new-id", b"revision-id")))
1385
self.assertEqual((b'file-id',), mapper.unmap("9b/file-id"))
1386
self.assertEqual((b'new-id',), mapper.unmap("45/new-id"))
1388
def test_hash_escaped_mapper(self):
1389
#knit1: hash + escaped
1390
mapper = versionedfile.HashEscapedPrefixMapper()
1391
self.assertEqual("88/%2520", mapper.map((b" ", b"revision-id")))
1392
self.assertEqual("ed/fil%2545-%2549d", mapper.map((b"filE-Id",
1394
self.assertEqual("88/ne%2557-%2549d", mapper.map((b"neW-Id",
1396
self.assertEqual((b'filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1397
self.assertEqual((b'neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1400
class TestVersionedFiles(TestCaseWithMemoryTransport):
1401
"""Tests for the multiple-file variant of VersionedFile."""
1403
# We want to be sure of behaviour for:
1404
# weaves prefix layout (weave texts)
1405
# individually named weaves (weave inventories)
1406
# annotated knits - prefix|hash|hash-escape layout, we test the third only
1407
# as it is the most complex mapper.
1408
# individually named knits
1409
# individual no-graph knits in packs (signatures)
1410
# individual graph knits in packs (inventories)
1411
# individual graph nocompression knits in packs (revisions)
1412
# plain text knits in packs (texts)
1413
len_one_scenarios = [
1416
'factory': make_versioned_files_factory(WeaveFile,
1417
ConstantMapper('inventory')),
1420
'support_partial_insertion': False,
1424
'factory': make_file_factory(False, ConstantMapper('revisions')),
1427
'support_partial_insertion': False,
1429
('named-nograph-nodelta-knit-pack', {
1430
'cleanup': cleanup_pack_knit,
1431
'factory': make_pack_factory(False, False, 1),
1434
'support_partial_insertion': False,
1436
('named-graph-knit-pack', {
1437
'cleanup': cleanup_pack_knit,
1438
'factory': make_pack_factory(True, True, 1),
1441
'support_partial_insertion': True,
1443
('named-graph-nodelta-knit-pack', {
1444
'cleanup': cleanup_pack_knit,
1445
'factory': make_pack_factory(True, False, 1),
1448
'support_partial_insertion': False,
1450
('groupcompress-nograph', {
1451
'cleanup': groupcompress.cleanup_pack_group,
1452
'factory': groupcompress.make_pack_factory(False, False, 1),
1455
'support_partial_insertion': False,
1458
len_two_scenarios = [
1461
'factory': make_versioned_files_factory(WeaveFile,
1465
'support_partial_insertion': False,
1467
('annotated-knit-escape', {
1469
'factory': make_file_factory(True, HashEscapedPrefixMapper()),
1472
'support_partial_insertion': False,
1474
('plain-knit-pack', {
1475
'cleanup': cleanup_pack_knit,
1476
'factory': make_pack_factory(True, True, 2),
1479
'support_partial_insertion': True,
1482
'cleanup': groupcompress.cleanup_pack_group,
1483
'factory': groupcompress.make_pack_factory(True, False, 1),
1486
'support_partial_insertion': False,
1490
scenarios = len_one_scenarios + len_two_scenarios
1492
def get_versionedfiles(self, relpath='files'):
1493
transport = self.get_transport(relpath)
1495
transport.mkdir('.')
1496
files = self.factory(transport)
1497
if self.cleanup is not None:
1498
self.addCleanup(self.cleanup, files)
1501
def get_simple_key(self, suffix):
1502
"""Return a key for the object under test."""
1503
if self.key_length == 1:
1506
return (b'FileA',) + (suffix,)
1508
def test_add_fallback_implies_without_fallbacks(self):
1509
f = self.get_versionedfiles('files')
1510
if getattr(f, 'add_fallback_versioned_files', None) is None:
1511
raise TestNotApplicable("%s doesn't support fallbacks"
1512
% (f.__class__.__name__,))
1513
g = self.get_versionedfiles('fallback')
1514
key_a = self.get_simple_key(b'a')
1515
g.add_lines(key_a, [], [b'\n'])
1516
f.add_fallback_versioned_files(g)
1517
self.assertTrue(key_a in f.get_parent_map([key_a]))
1519
key_a in f.without_fallbacks().get_parent_map([key_a]))
1521
def test_add_lines(self):
1522
f = self.get_versionedfiles()
1523
key0 = self.get_simple_key(b'r0')
1524
key1 = self.get_simple_key(b'r1')
1525
key2 = self.get_simple_key(b'r2')
1526
keyf = self.get_simple_key(b'foo')
1527
f.add_lines(key0, [], [b'a\n', b'b\n'])
1529
f.add_lines(key1, [key0], [b'b\n', b'c\n'])
1531
f.add_lines(key1, [], [b'b\n', b'c\n'])
1533
self.assertTrue(key0 in keys)
1534
self.assertTrue(key1 in keys)
1536
for record in f.get_record_stream([key0, key1], 'unordered', True):
1537
records.append((record.key, record.get_bytes_as('fulltext')))
1539
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1541
def test_add_chunks(self):
1542
f = self.get_versionedfiles()
1543
key0 = self.get_simple_key(b'r0')
1544
key1 = self.get_simple_key(b'r1')
1545
key2 = self.get_simple_key(b'r2')
1546
keyf = self.get_simple_key(b'foo')
1547
f.add_chunks(key0, [], [b'a', b'\nb\n'])
1549
f.add_chunks(key1, [key0], [b'b', b'\n', b'c\n'])
1551
f.add_chunks(key1, [], [b'b\n', b'c\n'])
1553
self.assertIn(key0, keys)
1554
self.assertIn(key1, keys)
1556
for record in f.get_record_stream([key0, key1], 'unordered', True):
1557
records.append((record.key, record.get_bytes_as('fulltext')))
1559
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1561
def test_annotate(self):
1562
files = self.get_versionedfiles()
1563
self.get_diamond_files(files)
1564
if self.key_length == 1:
1567
prefix = (b'FileA',)
1568
# introduced full text
1569
origins = files.annotate(prefix + (b'origin',))
1571
(prefix + (b'origin',), b'origin\n')],
1574
origins = files.annotate(prefix + (b'base',))
1576
(prefix + (b'base',), b'base\n')],
1579
origins = files.annotate(prefix + (b'merged',))
1582
(prefix + (b'base',), b'base\n'),
1583
(prefix + (b'left',), b'left\n'),
1584
(prefix + (b'right',), b'right\n'),
1585
(prefix + (b'merged',), b'merged\n')
1589
# Without a graph everything is new.
1591
(prefix + (b'merged',), b'base\n'),
1592
(prefix + (b'merged',), b'left\n'),
1593
(prefix + (b'merged',), b'right\n'),
1594
(prefix + (b'merged',), b'merged\n')
1597
self.assertRaises(RevisionNotPresent,
1598
files.annotate, prefix + ('missing-key',))
1600
def test_check_no_parameters(self):
1601
files = self.get_versionedfiles()
1603
def test_check_progressbar_parameter(self):
1604
"""A progress bar can be supplied because check can be a generator."""
1605
pb = ui.ui_factory.nested_progress_bar()
1606
self.addCleanup(pb.finished)
1607
files = self.get_versionedfiles()
1608
files.check(progress_bar=pb)
1610
def test_check_with_keys_becomes_generator(self):
1611
files = self.get_versionedfiles()
1612
self.get_diamond_files(files)
1614
entries = files.check(keys=keys)
1616
# Texts output should be fulltexts.
1617
self.capture_stream(files, entries, seen.add,
1618
files.get_parent_map(keys), require_fulltext=True)
1619
# All texts should be output.
1620
self.assertEqual(set(keys), seen)
1622
def test_clear_cache(self):
1623
files = self.get_versionedfiles()
1626
def test_construct(self):
1627
"""Each parameterised test can be constructed on a transport."""
1628
files = self.get_versionedfiles()
1630
def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1632
return get_diamond_files(files, self.key_length,
1633
trailing_eol=trailing_eol, nograph=not self.graph,
1634
left_only=left_only, nokeys=nokeys)
1636
def _add_content_nostoresha(self, add_lines):
1637
"""When nostore_sha is supplied using old content raises."""
1638
vf = self.get_versionedfiles()
1639
empty_text = (b'a', [])
1640
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
1641
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
1643
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1645
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1648
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1651
# we now have a copy of all the lines in the vf.
1652
for sha, (version, lines) in zip(
1653
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1654
new_key = self.get_simple_key(version + b"2")
1655
self.assertRaises(errors.ExistingContent,
1656
vf.add_lines, new_key, [], lines,
1658
self.assertRaises(errors.ExistingContent,
1659
vf.add_lines, new_key, [], lines,
1661
# and no new version should have been added.
1662
record = next(vf.get_record_stream([new_key], 'unordered', True))
1663
self.assertEqual('absent', record.storage_kind)
1665
def test_add_lines_nostoresha(self):
1666
self._add_content_nostoresha(add_lines=True)
1668
def test_add_lines_return(self):
1669
files = self.get_versionedfiles()
1670
# save code by using the stock data insertion helper.
1671
adds = self.get_diamond_files(files)
1673
# We can only validate the first 2 elements returned from add_lines.
1675
self.assertEqual(3, len(add))
1676
results.append(add[:2])
1677
if self.key_length == 1:
1679
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1680
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1681
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1682
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1683
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1685
elif self.key_length == 2:
1687
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1688
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1689
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1690
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1691
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1692
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1693
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1694
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1695
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1696
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1699
def test_add_lines_no_key_generates_chk_key(self):
1700
files = self.get_versionedfiles()
1701
# save code by using the stock data insertion helper.
1702
adds = self.get_diamond_files(files, nokeys=True)
1704
# We can only validate the first 2 elements returned from add_lines.
1706
self.assertEqual(3, len(add))
1707
results.append(add[:2])
1708
if self.key_length == 1:
1710
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1711
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1712
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1713
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1714
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1716
# Check the added items got CHK keys.
1718
(b'sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1719
(b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1720
(b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1721
(b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1722
(b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1725
elif self.key_length == 2:
1727
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1728
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1729
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1730
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1731
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1732
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1733
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1734
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1735
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1736
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1738
# Check the added items got CHK keys.
1740
(b'FileA', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1741
(b'FileA', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1742
(b'FileA', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1743
(b'FileA', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1744
(b'FileA', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1745
(b'FileB', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1746
(b'FileB', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1747
(b'FileB', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1748
(b'FileB', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1749
(b'FileB', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1753
def test_empty_lines(self):
1754
"""Empty files can be stored."""
1755
f = self.get_versionedfiles()
1756
key_a = self.get_simple_key(b'a')
1757
f.add_lines(key_a, [], [])
1758
self.assertEqual(b'',
1759
next(f.get_record_stream([key_a], 'unordered', True
1760
)).get_bytes_as('fulltext'))
1761
key_b = self.get_simple_key(b'b')
1762
f.add_lines(key_b, self.get_parents([key_a]), [])
1763
self.assertEqual(b'',
1764
next(f.get_record_stream([key_b], 'unordered', True
1765
)).get_bytes_as('fulltext'))
1767
def test_newline_only(self):
1768
f = self.get_versionedfiles()
1769
key_a = self.get_simple_key(b'a')
1770
f.add_lines(key_a, [], [b'\n'])
1771
self.assertEqual(b'\n',
1772
next(f.get_record_stream([key_a], 'unordered', True
1773
)).get_bytes_as('fulltext'))
1774
key_b = self.get_simple_key(b'b')
1775
f.add_lines(key_b, self.get_parents([key_a]), [b'\n'])
1776
self.assertEqual(b'\n',
1777
next(f.get_record_stream([key_b], 'unordered', True
1778
)).get_bytes_as('fulltext'))
1780
def test_get_known_graph_ancestry(self):
1781
f = self.get_versionedfiles()
1783
raise TestNotApplicable('ancestry info only relevant with graph.')
1784
key_a = self.get_simple_key(b'a')
1785
key_b = self.get_simple_key(b'b')
1786
key_c = self.get_simple_key(b'c')
1792
f.add_lines(key_a, [], [b'\n'])
1793
f.add_lines(key_b, [key_a], [b'\n'])
1794
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1795
kg = f.get_known_graph_ancestry([key_c])
1796
self.assertIsInstance(kg, _mod_graph.KnownGraph)
1797
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1799
def test_known_graph_with_fallbacks(self):
1800
f = self.get_versionedfiles('files')
1802
raise TestNotApplicable('ancestry info only relevant with graph.')
1803
if getattr(f, 'add_fallback_versioned_files', None) is None:
1804
raise TestNotApplicable("%s doesn't support fallbacks"
1805
% (f.__class__.__name__,))
1806
key_a = self.get_simple_key(b'a')
1807
key_b = self.get_simple_key(b'b')
1808
key_c = self.get_simple_key(b'c')
1809
# A only in fallback
1814
g = self.get_versionedfiles('fallback')
1815
g.add_lines(key_a, [], [b'\n'])
1816
f.add_fallback_versioned_files(g)
1817
f.add_lines(key_b, [key_a], [b'\n'])
1818
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1819
kg = f.get_known_graph_ancestry([key_c])
1820
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1822
def test_get_record_stream_empty(self):
1823
"""An empty stream can be requested without error."""
1824
f = self.get_versionedfiles()
1825
entries = f.get_record_stream([], 'unordered', False)
1826
self.assertEqual([], list(entries))
1828
def assertValidStorageKind(self, storage_kind):
1829
"""Assert that storage_kind is a valid storage_kind."""
1830
self.assertSubset([storage_kind],
1831
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1832
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1833
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1835
'knit-delta-closure', 'knit-delta-closure-ref',
1836
'groupcompress-block', 'groupcompress-block-ref'])
1838
def capture_stream(self, f, entries, on_seen, parents,
1839
require_fulltext=False):
1840
"""Capture a stream for testing."""
1841
for factory in entries:
1842
on_seen(factory.key)
1843
self.assertValidStorageKind(factory.storage_kind)
1844
if factory.sha1 is not None:
1845
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1847
self.assertEqual(parents[factory.key], factory.parents)
1848
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1850
if require_fulltext:
1851
factory.get_bytes_as('fulltext')
1853
def test_get_record_stream_interface(self):
1854
"""each item in a stream has to provide a regular interface."""
1855
files = self.get_versionedfiles()
1856
self.get_diamond_files(files)
1857
keys, _ = self.get_keys_and_sort_order()
1858
parent_map = files.get_parent_map(keys)
1859
entries = files.get_record_stream(keys, 'unordered', False)
1861
self.capture_stream(files, entries, seen.add, parent_map)
1862
self.assertEqual(set(keys), seen)
1864
def get_keys_and_sort_order(self):
1865
"""Get diamond test keys list, and their sort ordering."""
1866
if self.key_length == 1:
1867
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1868
sort_order = {(b'merged',): 2, (b'left',): 1,
1869
(b'right',): 1, (b'base',): 0}
1872
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1873
(b'FileA', b'base'),
1874
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1875
(b'FileB', b'base'),
1878
(b'FileA', b'merged'): 2, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1879
(b'FileA', b'base'): 0,
1880
(b'FileB', b'merged'): 2, (b'FileB', b'left'): 1, (b'FileB', b'right'): 1,
1881
(b'FileB', b'base'): 0,
1883
return keys, sort_order
1885
def get_keys_and_groupcompress_sort_order(self):
1886
"""Get diamond test keys list, and their groupcompress sort ordering."""
1887
if self.key_length == 1:
1888
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1889
sort_order = {(b'merged',): 0, (b'left',): 1,
1890
(b'right',): 1, (b'base',): 2}
1893
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1894
(b'FileA', b'base'),
1895
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1896
(b'FileB', b'base'),
1899
(b'FileA', b'merged'): 0, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1900
(b'FileA', b'base'): 2,
1901
(b'FileB', b'merged'): 3, (b'FileB', b'left'): 4, (b'FileB', b'right'): 4,
1902
(b'FileB', b'base'): 5,
1904
return keys, sort_order
1906
def test_get_record_stream_interface_ordered(self):
1907
"""each item in a stream has to provide a regular interface."""
1908
files = self.get_versionedfiles()
1909
self.get_diamond_files(files)
1910
keys, sort_order = self.get_keys_and_sort_order()
1911
parent_map = files.get_parent_map(keys)
1912
entries = files.get_record_stream(keys, 'topological', False)
1914
self.capture_stream(files, entries, seen.append, parent_map)
1915
self.assertStreamOrder(sort_order, seen, keys)
1917
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1918
"""each item must be accessible as a fulltext."""
1919
files = self.get_versionedfiles()
1920
self.get_diamond_files(files)
1921
keys, sort_order = self.get_keys_and_sort_order()
1922
parent_map = files.get_parent_map(keys)
1923
entries = files.get_record_stream(keys, 'topological', True)
1925
for factory in entries:
1926
seen.append(factory.key)
1927
self.assertValidStorageKind(factory.storage_kind)
1928
self.assertSubset([factory.sha1],
1929
[None, files.get_sha1s([factory.key])[factory.key]])
1930
self.assertEqual(parent_map[factory.key], factory.parents)
1931
# self.assertEqual(files.get_text(factory.key),
1932
ft_bytes = factory.get_bytes_as('fulltext')
1933
self.assertIsInstance(ft_bytes, bytes)
1934
chunked_bytes = factory.get_bytes_as('chunked')
1935
self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1937
self.assertStreamOrder(sort_order, seen, keys)
1939
def test_get_record_stream_interface_groupcompress(self):
1940
"""each item in a stream has to provide a regular interface."""
1941
files = self.get_versionedfiles()
1942
self.get_diamond_files(files)
1943
keys, sort_order = self.get_keys_and_groupcompress_sort_order()
1944
parent_map = files.get_parent_map(keys)
1945
entries = files.get_record_stream(keys, 'groupcompress', False)
1947
self.capture_stream(files, entries, seen.append, parent_map)
1948
self.assertStreamOrder(sort_order, seen, keys)
1950
def assertStreamOrder(self, sort_order, seen, keys):
1951
self.assertEqual(len(set(seen)), len(keys))
1952
if self.key_length == 1:
1955
lows = {(b'FileA',): 0, (b'FileB',): 0}
1957
self.assertEqual(set(keys), set(seen))
1960
sort_pos = sort_order[key]
1961
self.assertTrue(sort_pos >= lows[key[:-1]],
1962
"Out of order in sorted stream: %r, %r" % (key, seen))
1963
lows[key[:-1]] = sort_pos
1965
def test_get_record_stream_unknown_storage_kind_raises(self):
1966
"""Asking for a storage kind that the stream cannot supply raises."""
1967
files = self.get_versionedfiles()
1968
self.get_diamond_files(files)
1969
if self.key_length == 1:
1970
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1973
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1974
(b'FileA', b'base'),
1975
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1976
(b'FileB', b'base'),
1978
parent_map = files.get_parent_map(keys)
1979
entries = files.get_record_stream(keys, 'unordered', False)
1980
# We track the contents because we should be able to try, fail a
1981
# particular kind and then ask for one that works and continue.
1983
for factory in entries:
1984
seen.add(factory.key)
1985
self.assertValidStorageKind(factory.storage_kind)
1986
if factory.sha1 is not None:
1987
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1989
self.assertEqual(parent_map[factory.key], factory.parents)
1990
# currently no stream emits mpdiff
1991
self.assertRaises(errors.UnavailableRepresentation,
1992
factory.get_bytes_as, 'mpdiff')
1993
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1995
self.assertEqual(set(keys), seen)
1997
def test_get_record_stream_missing_records_are_absent(self):
1998
files = self.get_versionedfiles()
1999
self.get_diamond_files(files)
2000
if self.key_length == 1:
2001
keys = [(b'merged',), (b'left',), (b'right',),
2002
(b'absent',), (b'base',)]
2005
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
2006
(b'FileA', b'absent'), (b'FileA', b'base'),
2007
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
2008
(b'FileB', b'absent'), (b'FileB', b'base'),
2009
(b'absent', b'absent'),
2011
parent_map = files.get_parent_map(keys)
2012
entries = files.get_record_stream(keys, 'unordered', False)
2013
self.assertAbsentRecord(files, keys, parent_map, entries)
2014
entries = files.get_record_stream(keys, 'topological', False)
2015
self.assertAbsentRecord(files, keys, parent_map, entries)
2017
def assertRecordHasContent(self, record, bytes):
2018
"""Assert that record has the bytes bytes."""
2019
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
2020
self.assertEqual(bytes, b''.join(record.get_bytes_as('chunked')))
2022
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
2023
files = self.get_versionedfiles()
2024
key = self.get_simple_key(b'foo')
2025
files.add_lines(key, (), [b'my text\n', b'content'])
2026
stream = files.get_record_stream([key], 'unordered', False)
2027
record = next(stream)
2028
if record.storage_kind in ('chunked', 'fulltext'):
2029
# chunked and fulltext representations are for direct use not wire
2030
# serialisation: check they are able to be used directly. To send
2031
# such records over the wire translation will be needed.
2032
self.assertRecordHasContent(record, b"my text\ncontent")
2034
bytes = [record.get_bytes_as(record.storage_kind)]
2035
network_stream = versionedfile.NetworkRecordStream(bytes).read()
2036
source_record = record
2038
for record in network_stream:
2039
records.append(record)
2040
self.assertEqual(source_record.storage_kind,
2041
record.storage_kind)
2042
self.assertEqual(source_record.parents, record.parents)
2044
source_record.get_bytes_as(source_record.storage_kind),
2045
record.get_bytes_as(record.storage_kind))
2046
self.assertEqual(1, len(records))
2048
def assertStreamMetaEqual(self, records, expected, stream):
2049
"""Assert that streams expected and stream have the same records.
2051
:param records: A list to collect the seen records.
2052
:return: A generator of the records in stream.
2054
# We make assertions during copying to catch things early for easier
2055
# debugging. This must use the iterating zip() from the future.
2056
for record, ref_record in zip(stream, expected):
2057
records.append(record)
2058
self.assertEqual(ref_record.key, record.key)
2059
self.assertEqual(ref_record.storage_kind, record.storage_kind)
2060
self.assertEqual(ref_record.parents, record.parents)
2063
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
2065
"""Convert a stream to a bytes iterator.
2067
:param skipped_records: A list with one element to increment when a
2069
:param full_texts: A dict from key->fulltext representation, for
2070
checking chunked or fulltext stored records.
2071
:param stream: A record_stream.
2072
:return: An iterator over the bytes of each record.
2074
for record in stream:
2075
if record.storage_kind in ('chunked', 'fulltext'):
2076
skipped_records[0] += 1
2077
# check the content is correct for direct use.
2078
self.assertRecordHasContent(record, full_texts[record.key])
2080
yield record.get_bytes_as(record.storage_kind)
2082
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
2083
files = self.get_versionedfiles()
2084
target_files = self.get_versionedfiles('target')
2085
key = self.get_simple_key(b'ft')
2086
key_delta = self.get_simple_key(b'delta')
2087
files.add_lines(key, (), [b'my text\n', b'content'])
2089
delta_parents = (key,)
2092
files.add_lines(key_delta, delta_parents, [
2093
b'different\n', b'content\n'])
2094
local = files.get_record_stream([key, key_delta], 'unordered', False)
2095
ref = files.get_record_stream([key, key_delta], 'unordered', False)
2096
skipped_records = [0]
2098
key: b"my text\ncontent",
2099
key_delta: b"different\ncontent\n",
2101
byte_stream = self.stream_to_bytes_or_skip_counter(
2102
skipped_records, full_texts, local)
2103
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2105
# insert the stream from the network into a versioned files object so we can
2106
# check the content was carried across correctly without doing delta
2108
target_files.insert_record_stream(
2109
self.assertStreamMetaEqual(records, ref, network_stream))
2110
# No duplicates on the wire thank you!
2111
self.assertEqual(2, len(records) + skipped_records[0])
2113
# if any content was copied it all must have all been.
2114
self.assertIdenticalVersionedFile(files, target_files)
2116
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
2117
# copy a delta over the wire
2118
files = self.get_versionedfiles()
2119
target_files = self.get_versionedfiles('target')
2120
key = self.get_simple_key(b'ft')
2121
key_delta = self.get_simple_key(b'delta')
2122
files.add_lines(key, (), [b'my text\n', b'content'])
2124
delta_parents = (key,)
2127
files.add_lines(key_delta, delta_parents, [
2128
b'different\n', b'content\n'])
2129
# Copy the basis text across so we can reconstruct the delta during
2130
# insertion into target.
2131
target_files.insert_record_stream(files.get_record_stream([key],
2132
'unordered', False))
2133
local = files.get_record_stream([key_delta], 'unordered', False)
2134
ref = files.get_record_stream([key_delta], 'unordered', False)
2135
skipped_records = [0]
2137
key_delta: b"different\ncontent\n",
2139
byte_stream = self.stream_to_bytes_or_skip_counter(
2140
skipped_records, full_texts, local)
2141
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2143
# insert the stream from the network into a versioned files object so we can
2144
# check the content was carried across correctly without doing delta
2145
# inspection during check_stream.
2146
target_files.insert_record_stream(
2147
self.assertStreamMetaEqual(records, ref, network_stream))
2148
# No duplicates on the wire thank you!
2149
self.assertEqual(1, len(records) + skipped_records[0])
2151
# if any content was copied it all must have all been
2152
self.assertIdenticalVersionedFile(files, target_files)
2154
def test_get_record_stream_wire_ready_delta_closure_included(self):
2155
# copy a delta over the wire with the ability to get its full text.
2156
files = self.get_versionedfiles()
2157
key = self.get_simple_key(b'ft')
2158
key_delta = self.get_simple_key(b'delta')
2159
files.add_lines(key, (), [b'my text\n', b'content'])
2161
delta_parents = (key,)
2164
files.add_lines(key_delta, delta_parents, [
2165
b'different\n', b'content\n'])
2166
local = files.get_record_stream([key_delta], 'unordered', True)
2167
ref = files.get_record_stream([key_delta], 'unordered', True)
2168
skipped_records = [0]
2170
key_delta: b"different\ncontent\n",
2172
byte_stream = self.stream_to_bytes_or_skip_counter(
2173
skipped_records, full_texts, local)
2174
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2176
# insert the stream from the network into a versioned files object so we can
2177
# check the content was carried across correctly without doing delta
2178
# inspection during check_stream.
2179
for record in self.assertStreamMetaEqual(records, ref, network_stream):
2180
# we have to be able to get the full text out:
2181
self.assertRecordHasContent(record, full_texts[record.key])
2182
# No duplicates on the wire thank you!
2183
self.assertEqual(1, len(records) + skipped_records[0])
2185
def assertAbsentRecord(self, files, keys, parents, entries):
2186
"""Helper for test_get_record_stream_missing_records_are_absent."""
2188
for factory in entries:
2189
seen.add(factory.key)
2190
if factory.key[-1] == b'absent':
2191
self.assertEqual('absent', factory.storage_kind)
2192
self.assertEqual(None, factory.sha1)
2193
self.assertEqual(None, factory.parents)
2195
self.assertValidStorageKind(factory.storage_kind)
2196
if factory.sha1 is not None:
2197
sha1 = files.get_sha1s([factory.key])[factory.key]
2198
self.assertEqual(sha1, factory.sha1)
2199
self.assertEqual(parents[factory.key], factory.parents)
2200
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2202
self.assertEqual(set(keys), seen)
2204
def test_filter_absent_records(self):
2205
"""Requested missing records can be filter trivially."""
2206
files = self.get_versionedfiles()
2207
self.get_diamond_files(files)
2208
keys, _ = self.get_keys_and_sort_order()
2209
parent_map = files.get_parent_map(keys)
2210
# Add an absent record in the middle of the present keys. (We don't ask
2211
# for just absent keys to ensure that content before and after the
2212
# absent keys is still delivered).
2213
present_keys = list(keys)
2214
if self.key_length == 1:
2215
keys.insert(2, (b'extra',))
2217
keys.insert(2, (b'extra', b'extra'))
2218
entries = files.get_record_stream(keys, 'unordered', False)
2220
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2222
self.assertEqual(set(present_keys), seen)
2224
def get_mapper(self):
2225
"""Get a mapper suitable for the key length of the test interface."""
2226
if self.key_length == 1:
2227
return ConstantMapper('source')
2229
return HashEscapedPrefixMapper()
2231
def get_parents(self, parents):
2232
"""Get parents, taking self.graph into consideration."""
2238
def test_get_annotator(self):
2239
files = self.get_versionedfiles()
2240
self.get_diamond_files(files)
2241
origin_key = self.get_simple_key(b'origin')
2242
base_key = self.get_simple_key(b'base')
2243
left_key = self.get_simple_key(b'left')
2244
right_key = self.get_simple_key(b'right')
2245
merged_key = self.get_simple_key(b'merged')
2246
# annotator = files.get_annotator()
2247
# introduced full text
2248
origins, lines = files.get_annotator().annotate(origin_key)
2249
self.assertEqual([(origin_key,)], origins)
2250
self.assertEqual([b'origin\n'], lines)
2252
origins, lines = files.get_annotator().annotate(base_key)
2253
self.assertEqual([(base_key,)], origins)
2255
origins, lines = files.get_annotator().annotate(merged_key)
2264
# Without a graph everything is new.
2271
self.assertRaises(RevisionNotPresent,
2272
files.get_annotator().annotate, self.get_simple_key(b'missing-key'))
2274
def test_get_parent_map(self):
2275
files = self.get_versionedfiles()
2276
if self.key_length == 1:
2278
((b'r0',), self.get_parents(())),
2279
((b'r1',), self.get_parents(((b'r0',),))),
2280
((b'r2',), self.get_parents(())),
2281
((b'r3',), self.get_parents(())),
2282
((b'm',), self.get_parents(((b'r0',), (b'r1',), (b'r2',), (b'r3',)))),
2286
((b'FileA', b'r0'), self.get_parents(())),
2287
((b'FileA', b'r1'), self.get_parents(((b'FileA', b'r0'),))),
2288
((b'FileA', b'r2'), self.get_parents(())),
2289
((b'FileA', b'r3'), self.get_parents(())),
2290
((b'FileA', b'm'), self.get_parents(((b'FileA', b'r0'),
2291
(b'FileA', b'r1'), (b'FileA', b'r2'), (b'FileA', b'r3')))),
2293
for key, parents in parent_details:
2294
files.add_lines(key, parents, [])
2295
# immediately after adding it should be queryable.
2296
self.assertEqual({key: parents}, files.get_parent_map([key]))
2297
# We can ask for an empty set
2298
self.assertEqual({}, files.get_parent_map([]))
2299
# We can ask for many keys
2300
all_parents = dict(parent_details)
2301
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2302
# Absent keys are just not included in the result.
2303
keys = list(all_parents.keys())
2304
if self.key_length == 1:
2305
keys.insert(1, (b'missing',))
2307
keys.insert(1, (b'missing', b'missing'))
2308
# Absent keys are just ignored
2309
self.assertEqual(all_parents, files.get_parent_map(keys))
2311
def test_get_sha1s(self):
2312
files = self.get_versionedfiles()
2313
self.get_diamond_files(files)
2314
if self.key_length == 1:
2315
keys = [(b'base',), (b'origin',), (b'left',),
2316
(b'merged',), (b'right',)]
2318
# ask for shas from different prefixes.
2320
(b'FileA', b'base'), (b'FileB', b'origin'), (b'FileA', b'left'),
2321
(b'FileA', b'merged'), (b'FileB', b'right'),
2324
keys[0]: b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2325
keys[1]: b'00e364d235126be43292ab09cb4686cf703ddc17',
2326
keys[2]: b'a8478686da38e370e32e42e8a0c220e33ee9132f',
2327
keys[3]: b'ed8bce375198ea62444dc71952b22cfc2b09226d',
2328
keys[4]: b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2330
files.get_sha1s(keys))
2332
def test_insert_record_stream_empty(self):
2333
"""Inserting an empty record stream should work."""
2334
files = self.get_versionedfiles()
2335
files.insert_record_stream([])
2337
def assertIdenticalVersionedFile(self, expected, actual):
2338
"""Assert that left and right have the same contents."""
2339
self.assertEqual(set(actual.keys()), set(expected.keys()))
2340
actual_parents = actual.get_parent_map(actual.keys())
2343
actual_parents, expected.get_parent_map(expected.keys()))
2345
for key, parents in actual_parents.items():
2346
self.assertEqual(None, parents)
2347
for key in actual.keys():
2348
actual_text = next(actual.get_record_stream(
2349
[key], 'unordered', True)).get_bytes_as('fulltext')
2350
expected_text = next(expected.get_record_stream(
2351
[key], 'unordered', True)).get_bytes_as('fulltext')
2352
self.assertEqual(actual_text, expected_text)
2354
def test_insert_record_stream_fulltexts(self):
2355
"""Any file should accept a stream of fulltexts."""
2356
files = self.get_versionedfiles()
2357
mapper = self.get_mapper()
2358
source_transport = self.get_transport('source')
2359
source_transport.mkdir('.')
2360
# weaves always output fulltexts.
2361
source = make_versioned_files_factory(WeaveFile, mapper)(
2363
self.get_diamond_files(source, trailing_eol=False)
2364
stream = source.get_record_stream(source.keys(), 'topological',
2366
files.insert_record_stream(stream)
2367
self.assertIdenticalVersionedFile(source, files)
2369
def test_insert_record_stream_fulltexts_noeol(self):
2370
"""Any file should accept a stream of fulltexts."""
2371
files = self.get_versionedfiles()
2372
mapper = self.get_mapper()
2373
source_transport = self.get_transport('source')
2374
source_transport.mkdir('.')
2375
# weaves always output fulltexts.
2376
source = make_versioned_files_factory(WeaveFile, mapper)(
2378
self.get_diamond_files(source, trailing_eol=False)
2379
stream = source.get_record_stream(source.keys(), 'topological',
2381
files.insert_record_stream(stream)
2382
self.assertIdenticalVersionedFile(source, files)
2384
def test_insert_record_stream_annotated_knits(self):
2385
"""Any file should accept a stream from plain knits."""
2386
files = self.get_versionedfiles()
2387
mapper = self.get_mapper()
2388
source_transport = self.get_transport('source')
2389
source_transport.mkdir('.')
2390
source = make_file_factory(True, mapper)(source_transport)
2391
self.get_diamond_files(source)
2392
stream = source.get_record_stream(source.keys(), 'topological',
2394
files.insert_record_stream(stream)
2395
self.assertIdenticalVersionedFile(source, files)
2397
def test_insert_record_stream_annotated_knits_noeol(self):
2398
"""Any file should accept a stream from plain knits."""
2399
files = self.get_versionedfiles()
2400
mapper = self.get_mapper()
2401
source_transport = self.get_transport('source')
2402
source_transport.mkdir('.')
2403
source = make_file_factory(True, mapper)(source_transport)
2404
self.get_diamond_files(source, trailing_eol=False)
2405
stream = source.get_record_stream(source.keys(), 'topological',
2407
files.insert_record_stream(stream)
2408
self.assertIdenticalVersionedFile(source, files)
2410
def test_insert_record_stream_plain_knits(self):
2411
"""Any file should accept a stream from plain knits."""
2412
files = self.get_versionedfiles()
2413
mapper = self.get_mapper()
2414
source_transport = self.get_transport('source')
2415
source_transport.mkdir('.')
2416
source = make_file_factory(False, mapper)(source_transport)
2417
self.get_diamond_files(source)
2418
stream = source.get_record_stream(source.keys(), 'topological',
2420
files.insert_record_stream(stream)
2421
self.assertIdenticalVersionedFile(source, files)
2423
def test_insert_record_stream_plain_knits_noeol(self):
2424
"""Any file should accept a stream from plain knits."""
2425
files = self.get_versionedfiles()
2426
mapper = self.get_mapper()
2427
source_transport = self.get_transport('source')
2428
source_transport.mkdir('.')
2429
source = make_file_factory(False, mapper)(source_transport)
2430
self.get_diamond_files(source, trailing_eol=False)
2431
stream = source.get_record_stream(source.keys(), 'topological',
2433
files.insert_record_stream(stream)
2434
self.assertIdenticalVersionedFile(source, files)
2436
def test_insert_record_stream_existing_keys(self):
2437
"""Inserting keys already in a file should not error."""
2438
files = self.get_versionedfiles()
2439
source = self.get_versionedfiles('source')
2440
self.get_diamond_files(source)
2441
# insert some keys into f.
2442
self.get_diamond_files(files, left_only=True)
2443
stream = source.get_record_stream(source.keys(), 'topological',
2445
files.insert_record_stream(stream)
2446
self.assertIdenticalVersionedFile(source, files)
2448
def test_insert_record_stream_missing_keys(self):
2449
"""Inserting a stream with absent keys should raise an error."""
2450
files = self.get_versionedfiles()
2451
source = self.get_versionedfiles('source')
2452
stream = source.get_record_stream([(b'missing',) * self.key_length],
2453
'topological', False)
2454
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2457
def test_insert_record_stream_out_of_order(self):
2458
"""An out of order stream can either error or work."""
2459
files = self.get_versionedfiles()
2460
source = self.get_versionedfiles('source')
2461
self.get_diamond_files(source)
2462
if self.key_length == 1:
2463
origin_keys = [(b'origin',)]
2464
end_keys = [(b'merged',), (b'left',)]
2465
start_keys = [(b'right',), (b'base',)]
2467
origin_keys = [(b'FileA', b'origin'), (b'FileB', b'origin')]
2468
end_keys = [(b'FileA', b'merged',), (b'FileA', b'left',),
2469
(b'FileB', b'merged',), (b'FileB', b'left',)]
2470
start_keys = [(b'FileA', b'right',), (b'FileA', b'base',),
2471
(b'FileB', b'right',), (b'FileB', b'base',)]
2472
origin_entries = source.get_record_stream(
2473
origin_keys, 'unordered', False)
2474
end_entries = source.get_record_stream(end_keys, 'topological', False)
2475
start_entries = source.get_record_stream(
2476
start_keys, 'topological', False)
2477
entries = itertools.chain(origin_entries, end_entries, start_entries)
2479
files.insert_record_stream(entries)
2480
except RevisionNotPresent:
2481
# Must not have corrupted the file.
2484
self.assertIdenticalVersionedFile(source, files)
2486
def test_insert_record_stream_long_parent_chain_out_of_order(self):
2487
"""An out of order stream can either error or work."""
2489
raise TestNotApplicable('ancestry info only relevant with graph.')
2490
# Create a reasonably long chain of records based on each other, where
2491
# most will be deltas.
2492
source = self.get_versionedfiles('source')
2495
content = [(b'same same %d\n' % n) for n in range(500)]
2496
letters = b'abcdefghijklmnopqrstuvwxyz'
2497
for i in range(len(letters)):
2498
letter = letters[i:i + 1]
2499
key = (b'key-' + letter,)
2500
if self.key_length == 2:
2501
key = (b'prefix',) + key
2502
content.append(b'content for ' + letter + b'\n')
2503
source.add_lines(key, parents, content)
2506
# Create a stream of these records, excluding the first record that the
2507
# rest ultimately depend upon, and insert it into a new vf.
2509
for key in reversed(keys):
2510
streams.append(source.get_record_stream([key], 'unordered', False))
2511
deltas = itertools.chain.from_iterable(streams[:-1])
2512
files = self.get_versionedfiles()
2514
files.insert_record_stream(deltas)
2515
except RevisionNotPresent:
2516
# Must not have corrupted the file.
2519
# Must only report either just the first key as a missing parent,
2520
# no key as missing (for nodelta scenarios).
2521
missing = set(files.get_missing_compression_parent_keys())
2522
missing.discard(keys[0])
2523
self.assertEqual(set(), missing)
2525
def get_knit_delta_source(self):
2526
"""Get a source that can produce a stream with knit delta records,
2527
regardless of this test's scenario.
2529
mapper = self.get_mapper()
2530
source_transport = self.get_transport('source')
2531
source_transport.mkdir('.')
2532
source = make_file_factory(False, mapper)(source_transport)
2533
get_diamond_files(source, self.key_length, trailing_eol=True,
2534
nograph=False, left_only=False)
2537
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2538
"""Insertion where a needed basis is not included notifies the caller
2539
of the missing basis. In the meantime a record missing its basis is
2542
source = self.get_knit_delta_source()
2543
keys = [self.get_simple_key(b'origin'), self.get_simple_key(b'merged')]
2544
entries = source.get_record_stream(keys, 'unordered', False)
2545
files = self.get_versionedfiles()
2546
if self.support_partial_insertion:
2547
self.assertEqual([],
2548
list(files.get_missing_compression_parent_keys()))
2549
files.insert_record_stream(entries)
2550
missing_bases = files.get_missing_compression_parent_keys()
2551
self.assertEqual({self.get_simple_key(b'left')},
2553
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2556
errors.RevisionNotPresent, files.insert_record_stream, entries)
2559
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2560
"""Insertion where a needed basis is not included notifies the caller
2561
of the missing basis. That basis can be added in a second
2562
insert_record_stream call that does not need to repeat records present
2563
in the previous stream. The record(s) that required that basis are
2564
fully inserted once their basis is no longer missing.
2566
if not self.support_partial_insertion:
2567
raise TestNotApplicable(
2568
'versioned file scenario does not support partial insertion')
2569
source = self.get_knit_delta_source()
2570
entries = source.get_record_stream([self.get_simple_key(b'origin'),
2571
self.get_simple_key(b'merged')], 'unordered', False)
2572
files = self.get_versionedfiles()
2573
files.insert_record_stream(entries)
2574
missing_bases = files.get_missing_compression_parent_keys()
2575
self.assertEqual({self.get_simple_key(b'left')},
2577
# 'merged' is inserted (although a commit of a write group involving
2578
# this versionedfiles would fail).
2579
merged_key = self.get_simple_key(b'merged')
2581
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2582
# Add the full delta closure of the missing records
2583
missing_entries = source.get_record_stream(
2584
missing_bases, 'unordered', True)
2585
files.insert_record_stream(missing_entries)
2586
# Now 'merged' is fully inserted (and a commit would succeed).
2587
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2589
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2592
def test_iter_lines_added_or_present_in_keys(self):
2593
# test that we get at least an equalset of the lines added by
2594
# versions in the store.
2595
# the ordering here is to make a tree so that dumb searches have
2596
# more changes to muck up.
2598
class InstrumentedProgress(progress.ProgressTask):
2601
progress.ProgressTask.__init__(self)
2604
def update(self, msg=None, current=None, total=None):
2605
self.updates.append((msg, current, total))
2607
files = self.get_versionedfiles()
2608
# add a base to get included
2609
files.add_lines(self.get_simple_key(b'base'), (), [b'base\n'])
2610
# add a ancestor to be included on one side
2611
files.add_lines(self.get_simple_key(
2612
b'lancestor'), (), [b'lancestor\n'])
2613
# add a ancestor to be included on the other side
2614
files.add_lines(self.get_simple_key(b'rancestor'),
2615
self.get_parents([self.get_simple_key(b'base')]), [b'rancestor\n'])
2616
# add a child of rancestor with no eofile-nl
2617
files.add_lines(self.get_simple_key(b'child'),
2618
self.get_parents([self.get_simple_key(b'rancestor')]),
2619
[b'base\n', b'child\n'])
2620
# add a child of lancestor and base to join the two roots
2621
files.add_lines(self.get_simple_key(b'otherchild'),
2622
self.get_parents([self.get_simple_key(b'lancestor'),
2623
self.get_simple_key(b'base')]),
2624
[b'base\n', b'lancestor\n', b'otherchild\n'])
2626
def iter_with_keys(keys, expected):
2627
# now we need to see what lines are returned, and how often.
2629
progress = InstrumentedProgress()
2630
# iterate over the lines
2631
for line in files.iter_lines_added_or_present_in_keys(keys,
2633
lines.setdefault(line, 0)
2635
if [] != progress.updates:
2636
self.assertEqual(expected, progress.updates)
2638
lines = iter_with_keys(
2639
[self.get_simple_key(b'child'),
2640
self.get_simple_key(b'otherchild')],
2641
[('Walking content', 0, 2),
2642
('Walking content', 1, 2),
2643
('Walking content', 2, 2)])
2644
# we must see child and otherchild
2645
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2647
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2648
# we dont care if we got more than that.
2651
lines = iter_with_keys(files.keys(),
2652
[('Walking content', 0, 5),
2653
('Walking content', 1, 5),
2654
('Walking content', 2, 5),
2655
('Walking content', 3, 5),
2656
('Walking content', 4, 5),
2657
('Walking content', 5, 5)])
2658
# all lines must be seen at least once
2659
self.assertTrue(lines[(b'base\n', self.get_simple_key(b'base'))] > 0)
2661
lines[(b'lancestor\n', self.get_simple_key(b'lancestor'))] > 0)
2663
lines[(b'rancestor\n', self.get_simple_key(b'rancestor'))] > 0)
2664
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2666
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2668
def test_make_mpdiffs(self):
2669
from breezy import multiparent
2670
files = self.get_versionedfiles('source')
2671
# add texts that should trip the knit maximum delta chain threshold
2672
# as well as doing parallel chains of data in knits.
2673
# this is done by two chains of 25 insertions
2674
files.add_lines(self.get_simple_key(b'base'), [], [b'line\n'])
2675
files.add_lines(self.get_simple_key(b'noeol'),
2676
self.get_parents([self.get_simple_key(b'base')]), [b'line'])
2677
# detailed eol tests:
2678
# shared last line with parent no-eol
2679
files.add_lines(self.get_simple_key(b'noeolsecond'),
2680
self.get_parents([self.get_simple_key(b'noeol')]),
2681
[b'line\n', b'line'])
2682
# differing last line with parent, both no-eol
2683
files.add_lines(self.get_simple_key(b'noeolnotshared'),
2685
[self.get_simple_key(b'noeolsecond')]),
2686
[b'line\n', b'phone'])
2687
# add eol following a noneol parent, change content
2688
files.add_lines(self.get_simple_key(b'eol'),
2689
self.get_parents([self.get_simple_key(b'noeol')]), [b'phone\n'])
2690
# add eol following a noneol parent, no change content
2691
files.add_lines(self.get_simple_key(b'eolline'),
2692
self.get_parents([self.get_simple_key(b'noeol')]), [b'line\n'])
2693
# noeol with no parents:
2694
files.add_lines(self.get_simple_key(b'noeolbase'), [], [b'line'])
2695
# noeol preceeding its leftmost parent in the output:
2696
# this is done by making it a merge of two parents with no common
2697
# anestry: noeolbase and noeol with the
2698
# later-inserted parent the leftmost.
2699
files.add_lines(self.get_simple_key(b'eolbeforefirstparent'),
2700
self.get_parents([self.get_simple_key(b'noeolbase'),
2701
self.get_simple_key(b'noeol')]),
2703
# two identical eol texts
2704
files.add_lines(self.get_simple_key(b'noeoldup'),
2705
self.get_parents([self.get_simple_key(b'noeol')]), [b'line'])
2706
next_parent = self.get_simple_key(b'base')
2707
text_name = b'chain1-'
2709
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2710
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2711
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2712
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2713
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
2714
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
2715
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2716
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2717
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
2718
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2719
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2720
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2721
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
2722
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2723
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
2724
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2725
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
2726
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2727
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
2728
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
2729
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2730
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2731
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2732
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2733
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2734
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2736
for depth in range(26):
2737
new_version = self.get_simple_key(text_name + b'%d' % depth)
2738
text = text + [b'line\n']
2739
files.add_lines(new_version, self.get_parents([next_parent]), text)
2740
next_parent = new_version
2741
next_parent = self.get_simple_key(b'base')
2742
text_name = b'chain2-'
2744
for depth in range(26):
2745
new_version = self.get_simple_key(text_name + b'%d' % depth)
2746
text = text + [b'line\n']
2747
files.add_lines(new_version, self.get_parents([next_parent]), text)
2748
next_parent = new_version
2749
target = self.get_versionedfiles('target')
2750
for key in multiparent.topo_iter_keys(files, files.keys()):
2751
mpdiff = files.make_mpdiffs([key])[0]
2752
parents = files.get_parent_map([key])[key] or []
2754
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2755
self.assertEqualDiff(
2756
next(files.get_record_stream([key], 'unordered',
2757
True)).get_bytes_as('fulltext'),
2758
next(target.get_record_stream([key], 'unordered',
2759
True)).get_bytes_as('fulltext')
2762
def test_keys(self):
2763
# While use is discouraged, versions() is still needed by aspects of
2765
files = self.get_versionedfiles()
2766
self.assertEqual(set(), set(files.keys()))
2767
if self.key_length == 1:
2770
key = (b'foo', b'bar',)
2771
files.add_lines(key, (), [])
2772
self.assertEqual({key}, set(files.keys()))
2775
class VirtualVersionedFilesTests(TestCase):
2776
"""Basic tests for the VirtualVersionedFiles implementations."""
2778
def _get_parent_map(self, keys):
2781
if k in self._parent_map:
2782
ret[k] = self._parent_map[k]
2786
super(VirtualVersionedFilesTests, self).setUp()
2788
self._parent_map = {}
2789
self.texts = VirtualVersionedFiles(self._get_parent_map,
2792
def test_add_lines(self):
2793
self.assertRaises(NotImplementedError,
2794
self.texts.add_lines, b"foo", [], [])
2796
def test_add_mpdiffs(self):
2797
self.assertRaises(NotImplementedError,
2798
self.texts.add_mpdiffs, [])
2800
def test_check_noerrors(self):
2803
def test_insert_record_stream(self):
2804
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2807
def test_get_sha1s_nonexistent(self):
2808
self.assertEqual({}, self.texts.get_sha1s([(b"NONEXISTENT",)]))
2810
def test_get_sha1s(self):
2811
self._lines[b"key"] = [b"dataline1", b"dataline2"]
2812
self.assertEqual({(b"key",): osutils.sha_strings(self._lines[b"key"])},
2813
self.texts.get_sha1s([(b"key",)]))
2815
def test_get_parent_map(self):
2816
self._parent_map = {b"G": (b"A", b"B")}
2817
self.assertEqual({(b"G",): ((b"A",), (b"B",))},
2818
self.texts.get_parent_map([(b"G",), (b"L",)]))
2820
def test_get_record_stream(self):
2821
self._lines[b"A"] = [b"FOO", b"BAR"]
2822
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2824
self.assertEqual("chunked", record.storage_kind)
2825
self.assertEqual(b"FOOBAR", record.get_bytes_as("fulltext"))
2826
self.assertEqual([b"FOO", b"BAR"], record.get_bytes_as("chunked"))
2828
def test_get_record_stream_absent(self):
2829
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2831
self.assertEqual("absent", record.storage_kind)
2833
def test_iter_lines_added_or_present_in_keys(self):
2834
self._lines[b"A"] = [b"FOO", b"BAR"]
2835
self._lines[b"B"] = [b"HEY"]
2836
self._lines[b"C"] = [b"Alberta"]
2837
it = self.texts.iter_lines_added_or_present_in_keys([(b"A",), (b"B",)])
2838
self.assertEqual(sorted([(b"FOO", b"A"), (b"BAR", b"A"), (b"HEY", b"B")]),
2842
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2844
def get_ordering_vf(self, key_priority):
2845
builder = self.make_branch_builder('test')
2846
builder.start_series()
2847
builder.build_snapshot(None, [
2848
('add', ('', b'TREE_ROOT', 'directory', None))],
2850
builder.build_snapshot([b'A'], [], revision_id=b'B')
2851
builder.build_snapshot([b'B'], [], revision_id=b'C')
2852
builder.build_snapshot([b'C'], [], revision_id=b'D')
2853
builder.finish_series()
2854
b = builder.get_branch()
2856
self.addCleanup(b.unlock)
2857
vf = b.repository.inventories
2858
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2860
def test_get_empty(self):
2861
vf = self.get_ordering_vf({})
2862
self.assertEqual([], vf.calls)
2864
def test_get_record_stream_topological(self):
2865
vf = self.get_ordering_vf(
2866
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2867
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2868
keys = [r.key for r in vf.get_record_stream(request_keys,
2869
'topological', False)]
2870
# We should have gotten the keys in topological order
2871
self.assertEqual([(b'A',), (b'B',), (b'C',), (b'D',)], keys)
2872
# And recorded that the request was made
2873
self.assertEqual([('get_record_stream', request_keys, 'topological',
2876
def test_get_record_stream_ordered(self):
2877
vf = self.get_ordering_vf(
2878
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2879
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2880
keys = [r.key for r in vf.get_record_stream(request_keys,
2881
'unordered', False)]
2882
# They should be returned based on their priority
2883
self.assertEqual([(b'D',), (b'B',), (b'A',), (b'C',)], keys)
2884
# And the request recorded
2885
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2888
def test_get_record_stream_implicit_order(self):
2889
vf = self.get_ordering_vf({(b'B',): 2, (b'D',): 1})
2890
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2891
keys = [r.key for r in vf.get_record_stream(request_keys,
2892
'unordered', False)]
2893
# A and C are not in the map, so they get sorted to the front. A comes
2894
# before C alphabetically, so it comes back first
2895
self.assertEqual([(b'A',), (b'C',), (b'D',), (b'B',)], keys)
2896
# And the request recorded
2897
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
breezy/tests/per_versionedfile.py
