1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
24
from gzip import GzipFile
39
from ..errors import (
41
RevisionAlreadyPresent,
43
from ..bzr.knit import (
48
from ..sixish import (
54
TestCaseWithMemoryTransport,
58
from .http_utils import TestCaseWithWebserver
59
from ..transport.memory import MemoryTransport
60
from ..bzr import versionedfile as versionedfile
61
from ..bzr.versionedfile import (
62
ChunkedContentFactory,
64
HashEscapedPrefixMapper,
66
VirtualVersionedFiles,
67
make_versioned_files_factory,
69
from ..bzr.weave import (
73
from ..bzr.weavefile import write_weave
74
from .scenarios import load_tests_apply_scenarios
77
load_tests = load_tests_apply_scenarios
80
def get_diamond_vf(f, trailing_eol=True, left_only=False):
81
"""Get a diamond graph to exercise deltas and merges.
83
:param trailing_eol: If True end the last line with \n.
87
b'base': ((b'origin',),),
88
b'left': ((b'base',),),
89
b'right': ((b'base',),),
90
b'merged': ((b'left',), (b'right',)),
92
# insert a diamond graph to exercise deltas and merges.
97
f.add_lines(b'origin', [], [b'origin' + last_char])
98
f.add_lines(b'base', [b'origin'], [b'base' + last_char])
99
f.add_lines(b'left', [b'base'], [b'base\n', b'left' + last_char])
101
f.add_lines(b'right', [b'base'],
102
[b'base\n', b'right' + last_char])
103
f.add_lines(b'merged', [b'left', b'right'],
104
[b'base\n', b'left\n', b'right\n', b'merged' + last_char])
108
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
109
nograph=False, nokeys=False):
110
"""Get a diamond graph to exercise deltas and merges.
112
This creates a 5-node graph in files. If files supports 2-length keys two
113
graphs are made to exercise the support for multiple ids.
115
:param trailing_eol: If True end the last line with \n.
116
:param key_length: The length of keys in files. Currently supports length 1
118
:param left_only: If True do not add the right and merged nodes.
119
:param nograph: If True, do not provide parents to the add_lines calls;
120
this is useful for tests that need inserted data but have graphless
122
:param nokeys: If True, pass None is as the key for all insertions.
123
Currently implies nograph.
124
:return: The results of the add_lines calls.
131
prefixes = [(b'FileA',), (b'FileB',)]
132
# insert a diamond graph to exercise deltas and merges.
139
def get_parents(suffix_list):
143
result = [prefix + suffix for suffix in suffix_list]
151
# we loop over each key because that spreads the inserts across prefixes,
152
# which is how commit operates.
153
for prefix in prefixes:
154
result.append(files.add_lines(prefix + get_key(b'origin'), (),
155
[b'origin' + last_char]))
156
for prefix in prefixes:
157
result.append(files.add_lines(prefix + get_key(b'base'),
158
get_parents([(b'origin',)]), [b'base' + last_char]))
159
for prefix in prefixes:
160
result.append(files.add_lines(prefix + get_key(b'left'),
161
get_parents([(b'base',)]),
162
[b'base\n', b'left' + last_char]))
164
for prefix in prefixes:
165
result.append(files.add_lines(prefix + get_key(b'right'),
166
get_parents([(b'base',)]),
167
[b'base\n', b'right' + last_char]))
168
for prefix in prefixes:
169
result.append(files.add_lines(prefix + get_key(b'merged'),
171
[(b'left',), (b'right',)]),
172
[b'base\n', b'left\n', b'right\n', b'merged' + last_char]))
176
class VersionedFileTestMixIn(object):
177
"""A mixin test class for testing VersionedFiles.
179
This is not an adaptor-style test at this point because
180
theres no dynamic substitution of versioned file implementations,
181
they are strictly controlled by their owning repositories.
184
def get_transaction(self):
185
if not hasattr(self, '_transaction'):
186
self._transaction = None
187
return self._transaction
191
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
192
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
195
versions = f.versions()
196
self.assertTrue(b'r0' in versions)
197
self.assertTrue(b'r1' in versions)
198
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
199
self.assertEqual(f.get_text(b'r0'), b'a\nb\n')
200
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
201
self.assertEqual(2, len(f))
202
self.assertEqual(2, f.num_versions())
204
self.assertRaises(RevisionNotPresent,
205
f.add_lines, b'r2', [b'foo'], [])
206
self.assertRaises(RevisionAlreadyPresent,
207
f.add_lines, b'r1', [], [])
209
# this checks that reopen with create=True does not break anything.
210
f = self.reopen_file(create=True)
213
def test_adds_with_parent_texts(self):
216
_, _, parent_texts[b'r0'] = f.add_lines(b'r0', [], [b'a\n', b'b\n'])
218
_, _, parent_texts[b'r1'] = f.add_lines_with_ghosts(b'r1',
219
[b'r0', b'ghost'], [b'b\n', b'c\n'], parent_texts=parent_texts)
220
except NotImplementedError:
221
# if the format doesn't support ghosts, just add normally.
222
_, _, parent_texts[b'r1'] = f.add_lines(b'r1',
223
[b'r0'], [b'b\n', b'c\n'], parent_texts=parent_texts)
224
f.add_lines(b'r2', [b'r1'], [b'c\n', b'd\n'],
225
parent_texts=parent_texts)
226
self.assertNotEqual(None, parent_texts[b'r0'])
227
self.assertNotEqual(None, parent_texts[b'r1'])
230
versions = f.versions()
231
self.assertTrue(b'r0' in versions)
232
self.assertTrue(b'r1' in versions)
233
self.assertTrue(b'r2' in versions)
234
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
235
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
236
self.assertEqual(f.get_lines(b'r2'), [b'c\n', b'd\n'])
237
self.assertEqual(3, f.num_versions())
238
origins = f.annotate(b'r1')
239
self.assertEqual(origins[0][0], b'r0')
240
self.assertEqual(origins[1][0], b'r1')
241
origins = f.annotate(b'r2')
242
self.assertEqual(origins[0][0], b'r1')
243
self.assertEqual(origins[1][0], b'r2')
246
f = self.reopen_file()
249
def test_add_unicode_content(self):
250
# unicode content is not permitted in versioned files.
251
# versioned files version sequences of bytes only.
253
self.assertRaises(errors.BzrBadParameterUnicode,
254
vf.add_lines, b'a', [], [b'a\n', u'b\n', b'c\n'])
256
(errors.BzrBadParameterUnicode, NotImplementedError),
257
vf.add_lines_with_ghosts, b'a', [], [b'a\n', u'b\n', b'c\n'])
259
def test_add_follows_left_matching_blocks(self):
260
"""If we change left_matching_blocks, delta changes
262
Note: There are multiple correct deltas in this case, because
263
we start with 1 "a" and we get 3.
266
if isinstance(vf, WeaveFile):
267
raise TestSkipped("WeaveFile ignores left_matching_blocks")
268
vf.add_lines(b'1', [], [b'a\n'])
269
vf.add_lines(b'2', [b'1'], [b'a\n', b'a\n', b'a\n'],
270
left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
271
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'2'))
272
vf.add_lines(b'3', [b'1'], [b'a\n', b'a\n', b'a\n'],
273
left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
274
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'3'))
276
def test_inline_newline_throws(self):
277
# \r characters are not permitted in lines being added
279
self.assertRaises(errors.BzrBadParameterContainsNewline,
280
vf.add_lines, b'a', [], [b'a\n\n'])
282
(errors.BzrBadParameterContainsNewline, NotImplementedError),
283
vf.add_lines_with_ghosts, b'a', [], [b'a\n\n'])
284
# but inline CR's are allowed
285
vf.add_lines(b'a', [], [b'a\r\n'])
287
vf.add_lines_with_ghosts(b'b', [], [b'a\r\n'])
288
except NotImplementedError:
291
def test_add_reserved(self):
293
self.assertRaises(errors.ReservedId,
294
vf.add_lines, b'a:', [], [b'a\n', b'b\n', b'c\n'])
296
def test_add_lines_nostoresha(self):
297
"""When nostore_sha is supplied using old content raises."""
299
empty_text = (b'a', [])
300
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
301
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
303
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
304
sha, _, _ = vf.add_lines(version, [], lines)
306
# we now have a copy of all the lines in the vf.
307
for sha, (version, lines) in zip(
308
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
309
self.assertRaises(errors.ExistingContent,
310
vf.add_lines, version + b"2", [], lines,
312
# and no new version should have been added.
313
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
316
def test_add_lines_with_ghosts_nostoresha(self):
317
"""When nostore_sha is supplied using old content raises."""
319
empty_text = (b'a', [])
320
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
321
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
323
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
324
sha, _, _ = vf.add_lines(version, [], lines)
326
# we now have a copy of all the lines in the vf.
327
# is the test applicable to this vf implementation?
329
vf.add_lines_with_ghosts(b'd', [], [])
330
except NotImplementedError:
331
raise TestSkipped("add_lines_with_ghosts is optional")
332
for sha, (version, lines) in zip(
333
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
334
self.assertRaises(errors.ExistingContent,
335
vf.add_lines_with_ghosts, version + b"2", [], lines,
337
# and no new version should have been added.
338
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
341
def test_add_lines_return_value(self):
342
# add_lines should return the sha1 and the text size.
344
empty_text = (b'a', [])
345
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
346
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
347
# check results for the three cases:
348
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
349
# the first two elements are the same for all versioned files:
350
# - the digest and the size of the text. For some versioned files
351
# additional data is returned in additional tuple elements.
352
result = vf.add_lines(version, [], lines)
353
self.assertEqual(3, len(result))
354
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
356
# parents should not affect the result:
357
lines = sample_text_nl[1]
358
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
359
vf.add_lines(b'd', [b'b', b'c'], lines)[0:2])
361
def test_get_reserved(self):
363
self.assertRaises(errors.ReservedId, vf.get_texts, [b'b:'])
364
self.assertRaises(errors.ReservedId, vf.get_lines, b'b:')
365
self.assertRaises(errors.ReservedId, vf.get_text, b'b:')
367
def test_add_unchanged_last_line_noeol_snapshot(self):
368
"""Add a text with an unchanged last line with no eol should work."""
369
# Test adding this in a number of chain lengths; because the interface
370
# for VersionedFile does not allow forcing a specific chain length, we
371
# just use a small base to get the first snapshot, then a much longer
372
# first line for the next add (which will make the third add snapshot)
373
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
374
# as a capped delta length, but ideally we would have some way of
375
# tuning the test to the store (e.g. keep going until a snapshot
377
for length in range(20):
379
vf = self.get_file('case-%d' % length)
382
for step in range(length):
383
version = prefix % step
384
lines = ([b'prelude \n'] * step) + [b'line']
385
vf.add_lines(version, parents, lines)
386
version_lines[version] = lines
388
vf.add_lines(b'no-eol', parents, [b'line'])
389
vf.get_texts(version_lines.keys())
390
self.assertEqualDiff(b'line', vf.get_text(b'no-eol'))
392
def test_get_texts_eol_variation(self):
393
# similar to the failure in <http://bugs.launchpad.net/234748>
395
sample_text_nl = [b"line\n"]
396
sample_text_no_nl = [b"line"]
403
lines = sample_text_nl
405
lines = sample_text_no_nl
406
# left_matching blocks is an internal api; it operates on the
407
# *internal* representation for a knit, which is with *all* lines
408
# being normalised to end with \n - even the final line in a no_nl
409
# file. Using it here ensures that a broken internal implementation
410
# (which is what this test tests) will generate a correct line
411
# delta (which is to say, an empty delta).
412
vf.add_lines(version, parents, lines,
413
left_matching_blocks=[(0, 0, 1)])
415
versions.append(version)
416
version_lines[version] = lines
418
vf.get_texts(versions)
419
vf.get_texts(reversed(versions))
421
def test_add_lines_with_matching_blocks_noeol_last_line(self):
422
"""Add a text with an unchanged last line with no eol should work."""
423
from breezy import multiparent
424
# Hand verified sha1 of the text we're adding.
425
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
426
# Create a mpdiff which adds a new line before the trailing line, and
427
# reuse the last line unaltered (which can cause annotation reuse).
428
# Test adding this in two situations:
429
# On top of a new insertion
430
vf = self.get_file('fulltext')
431
vf.add_lines(b'noeol', [], [b'line'])
432
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
433
left_matching_blocks=[(0, 1, 1)])
434
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
436
vf = self.get_file('delta')
437
vf.add_lines(b'base', [], [b'line'])
438
vf.add_lines(b'noeol', [b'base'], [b'prelude\n', b'line'])
439
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
440
left_matching_blocks=[(1, 1, 1)])
441
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
443
def test_make_mpdiffs(self):
444
from breezy import multiparent
445
vf = self.get_file('foo')
446
sha1s = self._setup_for_deltas(vf)
447
new_vf = self.get_file('bar')
448
for version in multiparent.topo_iter(vf):
449
mpdiff = vf.make_mpdiffs([version])[0]
450
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
451
vf.get_sha1s([version])[version], mpdiff)])
452
self.assertEqualDiff(vf.get_text(version),
453
new_vf.get_text(version))
455
def test_make_mpdiffs_with_ghosts(self):
456
vf = self.get_file('foo')
458
vf.add_lines_with_ghosts(b'text', [b'ghost'], [b'line\n'])
459
except NotImplementedError:
460
# old Weave formats do not allow ghosts
462
self.assertRaises(errors.RevisionNotPresent,
463
vf.make_mpdiffs, [b'ghost'])
465
def _setup_for_deltas(self, f):
466
self.assertFalse(f.has_version('base'))
467
# add texts that should trip the knit maximum delta chain threshold
468
# as well as doing parallel chains of data in knits.
469
# this is done by two chains of 25 insertions
470
f.add_lines(b'base', [], [b'line\n'])
471
f.add_lines(b'noeol', [b'base'], [b'line'])
472
# detailed eol tests:
473
# shared last line with parent no-eol
474
f.add_lines(b'noeolsecond', [b'noeol'], [b'line\n', b'line'])
475
# differing last line with parent, both no-eol
476
f.add_lines(b'noeolnotshared', [b'noeolsecond'], [b'line\n', b'phone'])
477
# add eol following a noneol parent, change content
478
f.add_lines(b'eol', [b'noeol'], [b'phone\n'])
479
# add eol following a noneol parent, no change content
480
f.add_lines(b'eolline', [b'noeol'], [b'line\n'])
481
# noeol with no parents:
482
f.add_lines(b'noeolbase', [], [b'line'])
483
# noeol preceeding its leftmost parent in the output:
484
# this is done by making it a merge of two parents with no common
485
# anestry: noeolbase and noeol with the
486
# later-inserted parent the leftmost.
487
f.add_lines(b'eolbeforefirstparent', [
488
b'noeolbase', b'noeol'], [b'line'])
489
# two identical eol texts
490
f.add_lines(b'noeoldup', [b'noeol'], [b'line'])
491
next_parent = b'base'
492
text_name = b'chain1-'
494
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
495
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
496
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
497
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
498
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
499
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
500
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
501
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
502
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
503
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
504
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
505
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
506
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
507
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
508
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
509
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
510
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
511
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
512
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
513
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
514
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
515
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
516
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
517
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
518
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
519
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
521
for depth in range(26):
522
new_version = text_name + b'%d' % depth
523
text = text + [b'line\n']
524
f.add_lines(new_version, [next_parent], text)
525
next_parent = new_version
526
next_parent = b'base'
527
text_name = b'chain2-'
529
for depth in range(26):
530
new_version = text_name + b'%d' % depth
531
text = text + [b'line\n']
532
f.add_lines(new_version, [next_parent], text)
533
next_parent = new_version
536
def test_ancestry(self):
538
self.assertEqual([], f.get_ancestry([]))
539
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
540
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
541
f.add_lines(b'r2', [b'r0'], [b'b\n', b'c\n'])
542
f.add_lines(b'r3', [b'r2'], [b'b\n', b'c\n'])
543
f.add_lines(b'rM', [b'r1', b'r2'], [b'b\n', b'c\n'])
544
self.assertEqual([], f.get_ancestry([]))
545
versions = f.get_ancestry([b'rM'])
546
# there are some possibilities:
550
# so we check indexes
551
r0 = versions.index(b'r0')
552
r1 = versions.index(b'r1')
553
r2 = versions.index(b'r2')
554
self.assertFalse(b'r3' in versions)
555
rM = versions.index(b'rM')
556
self.assertTrue(r0 < r1)
557
self.assertTrue(r0 < r2)
558
self.assertTrue(r1 < rM)
559
self.assertTrue(r2 < rM)
561
self.assertRaises(RevisionNotPresent,
562
f.get_ancestry, [b'rM', b'rX'])
564
self.assertEqual(set(f.get_ancestry(b'rM')),
565
set(f.get_ancestry(b'rM', topo_sorted=False)))
567
def test_mutate_after_finish(self):
568
self._transaction = 'before'
570
self._transaction = 'after'
571
self.assertRaises(errors.OutSideTransaction, f.add_lines, b'', [], [])
572
self.assertRaises(errors.OutSideTransaction,
573
f.add_lines_with_ghosts, b'', [], [])
575
def test_copy_to(self):
577
f.add_lines(b'0', [], [b'a\n'])
578
t = MemoryTransport()
580
for suffix in self.get_factory().get_suffixes():
581
self.assertTrue(t.has('foo' + suffix))
583
def test_get_suffixes(self):
585
# and should be a list
586
self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
588
def test_get_parent_map(self):
590
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
592
{b'r0': ()}, f.get_parent_map([b'r0']))
593
f.add_lines(b'r1', [b'r0'], [b'a\n', b'b\n'])
595
{b'r1': (b'r0',)}, f.get_parent_map([b'r1']))
599
f.get_parent_map([b'r0', b'r1']))
600
f.add_lines(b'r2', [], [b'a\n', b'b\n'])
601
f.add_lines(b'r3', [], [b'a\n', b'b\n'])
602
f.add_lines(b'm', [b'r0', b'r1', b'r2', b'r3'], [b'a\n', b'b\n'])
604
{b'm': (b'r0', b'r1', b'r2', b'r3')}, f.get_parent_map([b'm']))
605
self.assertEqual({}, f.get_parent_map(b'y'))
609
f.get_parent_map([b'r0', b'y', b'r1']))
611
def test_annotate(self):
613
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
614
f.add_lines(b'r1', [b'r0'], [b'c\n', b'b\n'])
615
origins = f.annotate(b'r1')
616
self.assertEqual(origins[0][0], b'r1')
617
self.assertEqual(origins[1][0], b'r0')
619
self.assertRaises(RevisionNotPresent,
622
def test_detection(self):
623
# Test weaves detect corruption.
625
# Weaves contain a checksum of their texts.
626
# When a text is extracted, this checksum should be
629
w = self.get_file_corrupted_text()
631
self.assertEqual(b'hello\n', w.get_text(b'v1'))
632
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
633
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
634
self.assertRaises(WeaveInvalidChecksum, w.check)
636
w = self.get_file_corrupted_checksum()
638
self.assertEqual(b'hello\n', w.get_text(b'v1'))
639
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
640
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
641
self.assertRaises(WeaveInvalidChecksum, w.check)
643
def get_file_corrupted_text(self):
644
"""Return a versioned file with corrupt text but valid metadata."""
645
raise NotImplementedError(self.get_file_corrupted_text)
647
def reopen_file(self, name='foo'):
648
"""Open the versioned file from disk again."""
649
raise NotImplementedError(self.reopen_file)
651
def test_iter_lines_added_or_present_in_versions(self):
652
# test that we get at least an equalset of the lines added by
653
# versions in the weave
654
# the ordering here is to make a tree so that dumb searches have
655
# more changes to muck up.
657
class InstrumentedProgress(progress.ProgressTask):
660
progress.ProgressTask.__init__(self)
663
def update(self, msg=None, current=None, total=None):
664
self.updates.append((msg, current, total))
667
# add a base to get included
668
vf.add_lines(b'base', [], [b'base\n'])
669
# add a ancestor to be included on one side
670
vf.add_lines(b'lancestor', [], [b'lancestor\n'])
671
# add a ancestor to be included on the other side
672
vf.add_lines(b'rancestor', [b'base'], [b'rancestor\n'])
673
# add a child of rancestor with no eofile-nl
674
vf.add_lines(b'child', [b'rancestor'], [b'base\n', b'child\n'])
675
# add a child of lancestor and base to join the two roots
676
vf.add_lines(b'otherchild',
677
[b'lancestor', b'base'],
678
[b'base\n', b'lancestor\n', b'otherchild\n'])
680
def iter_with_versions(versions, expected):
681
# now we need to see what lines are returned, and how often.
683
progress = InstrumentedProgress()
684
# iterate over the lines
685
for line in vf.iter_lines_added_or_present_in_versions(versions,
687
lines.setdefault(line, 0)
689
if [] != progress.updates:
690
self.assertEqual(expected, progress.updates)
692
lines = iter_with_versions([b'child', b'otherchild'],
693
[('Walking content', 0, 2),
694
('Walking content', 1, 2),
695
('Walking content', 2, 2)])
696
# we must see child and otherchild
697
self.assertTrue(lines[(b'child\n', b'child')] > 0)
698
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
699
# we dont care if we got more than that.
702
lines = iter_with_versions(None, [('Walking content', 0, 5),
703
('Walking content', 1, 5),
704
('Walking content', 2, 5),
705
('Walking content', 3, 5),
706
('Walking content', 4, 5),
707
('Walking content', 5, 5)])
708
# all lines must be seen at least once
709
self.assertTrue(lines[(b'base\n', b'base')] > 0)
710
self.assertTrue(lines[(b'lancestor\n', b'lancestor')] > 0)
711
self.assertTrue(lines[(b'rancestor\n', b'rancestor')] > 0)
712
self.assertTrue(lines[(b'child\n', b'child')] > 0)
713
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
715
def test_add_lines_with_ghosts(self):
716
# some versioned file formats allow lines to be added with parent
717
# information that is > than that in the format. Formats that do
718
# not support this need to raise NotImplementedError on the
719
# add_lines_with_ghosts api.
721
# add a revision with ghost parents
722
# The preferred form is utf8, but we should translate when needed
723
parent_id_unicode = u'b\xbfse'
724
parent_id_utf8 = parent_id_unicode.encode('utf8')
726
vf.add_lines_with_ghosts(b'notbxbfse', [parent_id_utf8], [])
727
except NotImplementedError:
728
# check the other ghost apis are also not implemented
729
self.assertRaises(NotImplementedError,
730
vf.get_ancestry_with_ghosts, [b'foo'])
731
self.assertRaises(NotImplementedError,
732
vf.get_parents_with_ghosts, b'foo')
734
vf = self.reopen_file()
735
# test key graph related apis: getncestry, _graph, get_parents
737
# - these are ghost unaware and must not be reflect ghosts
738
self.assertEqual([b'notbxbfse'], vf.get_ancestry(b'notbxbfse'))
739
self.assertFalse(vf.has_version(parent_id_utf8))
740
# we have _with_ghost apis to give us ghost information.
741
self.assertEqual([parent_id_utf8, b'notbxbfse'],
742
vf.get_ancestry_with_ghosts([b'notbxbfse']))
743
self.assertEqual([parent_id_utf8],
744
vf.get_parents_with_ghosts(b'notbxbfse'))
745
# if we add something that is a ghost of another, it should correct the
746
# results of the prior apis
747
vf.add_lines(parent_id_utf8, [], [])
748
self.assertEqual([parent_id_utf8, b'notbxbfse'],
749
vf.get_ancestry([b'notbxbfse']))
750
self.assertEqual({b'notbxbfse': (parent_id_utf8,)},
751
vf.get_parent_map([b'notbxbfse']))
752
self.assertTrue(vf.has_version(parent_id_utf8))
753
# we have _with_ghost apis to give us ghost information.
754
self.assertEqual([parent_id_utf8, b'notbxbfse'],
755
vf.get_ancestry_with_ghosts([b'notbxbfse']))
756
self.assertEqual([parent_id_utf8],
757
vf.get_parents_with_ghosts(b'notbxbfse'))
759
def test_add_lines_with_ghosts_after_normal_revs(self):
760
# some versioned file formats allow lines to be added with parent
761
# information that is > than that in the format. Formats that do
762
# not support this need to raise NotImplementedError on the
763
# add_lines_with_ghosts api.
765
# probe for ghost support
767
vf.add_lines_with_ghosts(b'base', [], [b'line\n', b'line_b\n'])
768
except NotImplementedError:
770
vf.add_lines_with_ghosts(b'references_ghost',
771
[b'base', b'a_ghost'],
772
[b'line\n', b'line_b\n', b'line_c\n'])
773
origins = vf.annotate(b'references_ghost')
774
self.assertEqual((b'base', b'line\n'), origins[0])
775
self.assertEqual((b'base', b'line_b\n'), origins[1])
776
self.assertEqual((b'references_ghost', b'line_c\n'), origins[2])
778
def test_readonly_mode(self):
779
t = self.get_transport()
780
factory = self.get_factory()
781
vf = factory('id', t, 0o777, create=True, access_mode='w')
782
vf = factory('id', t, access_mode='r')
783
self.assertRaises(errors.ReadOnlyError, vf.add_lines, b'base', [], [])
784
self.assertRaises(errors.ReadOnlyError,
785
vf.add_lines_with_ghosts,
790
def test_get_sha1s(self):
791
# check the sha1 data is available
794
vf.add_lines(b'a', [], [b'a\n'])
795
# the same file, different metadata
796
vf.add_lines(b'b', [b'a'], [b'a\n'])
797
# a file differing only in last newline.
798
vf.add_lines(b'c', [], [b'a'])
800
b'a': b'3f786850e387550fdab836ed7e6dc881de23001b',
801
b'c': b'86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
802
b'b': b'3f786850e387550fdab836ed7e6dc881de23001b',
804
vf.get_sha1s([b'a', b'c', b'b']))
807
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
809
def get_file(self, name='foo'):
810
return WeaveFile(name, self.get_transport(),
812
get_scope=self.get_transaction)
814
def get_file_corrupted_text(self):
815
w = WeaveFile('foo', self.get_transport(),
817
get_scope=self.get_transaction)
818
w.add_lines(b'v1', [], [b'hello\n'])
819
w.add_lines(b'v2', [b'v1'], [b'hello\n', b'there\n'])
821
# We are going to invasively corrupt the text
822
# Make sure the internals of weave are the same
823
self.assertEqual([(b'{', 0), b'hello\n', (b'}', None), (b'{', 1), b'there\n', (b'}', None)
826
self.assertEqual([b'f572d396fae9206628714fb2ce00f72e94f2258f', b'90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
831
w._weave[4] = b'There\n'
834
def get_file_corrupted_checksum(self):
835
w = self.get_file_corrupted_text()
837
w._weave[4] = b'there\n'
838
self.assertEqual(b'hello\nthere\n', w.get_text(b'v2'))
840
# Invalid checksum, first digit changed
841
w._sha1s[1] = b'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
844
def reopen_file(self, name='foo', create=False):
845
return WeaveFile(name, self.get_transport(),
847
get_scope=self.get_transaction)
849
def test_no_implicit_create(self):
850
self.assertRaises(errors.NoSuchFile,
853
self.get_transport(),
854
get_scope=self.get_transaction)
856
def get_factory(self):
860
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
863
super(TestPlanMergeVersionedFile, self).setUp()
864
mapper = PrefixMapper()
865
factory = make_file_factory(True, mapper)
866
self.vf1 = factory(self.get_transport('root-1'))
867
self.vf2 = factory(self.get_transport('root-2'))
868
self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
869
self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
871
def test_add_lines(self):
872
self.plan_merge_vf.add_lines((b'root', b'a:'), [], [])
873
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
874
(b'root', b'a'), [], [])
875
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
876
(b'root', b'a:'), None, [])
877
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
878
(b'root', b'a:'), [], None)
880
def setup_abcde(self):
881
self.vf1.add_lines((b'root', b'A'), [], [b'a'])
882
self.vf1.add_lines((b'root', b'B'), [(b'root', b'A')], [b'b'])
883
self.vf2.add_lines((b'root', b'C'), [], [b'c'])
884
self.vf2.add_lines((b'root', b'D'), [(b'root', b'C')], [b'd'])
885
self.plan_merge_vf.add_lines((b'root', b'E:'),
886
[(b'root', b'B'), (b'root', b'D')], [b'e'])
888
def test_get_parents(self):
890
self.assertEqual({(b'root', b'B'): ((b'root', b'A'),)},
891
self.plan_merge_vf.get_parent_map([(b'root', b'B')]))
892
self.assertEqual({(b'root', b'D'): ((b'root', b'C'),)},
893
self.plan_merge_vf.get_parent_map([(b'root', b'D')]))
894
self.assertEqual({(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D'))},
895
self.plan_merge_vf.get_parent_map([(b'root', b'E:')]))
897
self.plan_merge_vf.get_parent_map([(b'root', b'F')]))
899
(b'root', b'B'): ((b'root', b'A'),),
900
(b'root', b'D'): ((b'root', b'C'),),
901
(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D')),
903
self.plan_merge_vf.get_parent_map(
904
[(b'root', b'B'), (b'root', b'D'), (b'root', b'E:'), (b'root', b'F')]))
906
def test_get_record_stream(self):
909
def get_record(suffix):
910
return next(self.plan_merge_vf.get_record_stream(
911
[(b'root', suffix)], 'unordered', True))
912
self.assertEqual(b'a', get_record(b'A').get_bytes_as('fulltext'))
913
self.assertEqual(b'a', b''.join(get_record(b'A').iter_bytes_as('chunked')))
914
self.assertEqual(b'c', get_record(b'C').get_bytes_as('fulltext'))
915
self.assertEqual(b'e', get_record(b'E:').get_bytes_as('fulltext'))
916
self.assertEqual('absent', get_record('F').storage_kind)
919
class TestReadonlyHttpMixin(object):
921
def get_transaction(self):
924
def test_readonly_http_works(self):
925
# we should be able to read from http with a versioned file.
927
# try an empty file access
928
readonly_vf = self.get_factory()('foo',
929
transport.get_transport_from_url(self.get_readonly_url('.')))
930
self.assertEqual([], readonly_vf.versions())
932
def test_readonly_http_works_with_feeling(self):
933
# we should be able to read from http with a versioned file.
936
vf.add_lines(b'1', [], [b'a\n'])
937
vf.add_lines(b'2', [b'1'], [b'b\n', b'a\n'])
938
readonly_vf = self.get_factory()('foo',
939
transport.get_transport_from_url(self.get_readonly_url('.')))
940
self.assertEqual([b'1', b'2'], vf.versions())
941
self.assertEqual([b'1', b'2'], readonly_vf.versions())
942
for version in readonly_vf.versions():
943
readonly_vf.get_lines(version)
946
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
949
return WeaveFile('foo', self.get_transport(),
951
get_scope=self.get_transaction)
953
def get_factory(self):
957
class MergeCasesMixin(object):
959
def doMerge(self, base, a, b, mp):
960
from textwrap import dedent
966
w.add_lines(b'text0', [], list(map(addcrlf, base)))
967
w.add_lines(b'text1', [b'text0'], list(map(addcrlf, a)))
968
w.add_lines(b'text2', [b'text0'], list(map(addcrlf, b)))
972
self.log('merge plan:')
973
p = list(w.plan_merge(b'text1', b'text2'))
974
for state, line in p:
976
self.log('%12s | %s' % (state, line[:-1]))
980
mt.writelines(w.weave_merge(p))
982
self.log(mt.getvalue())
984
mp = list(map(addcrlf, mp))
985
self.assertEqual(mt.readlines(), mp)
987
def testOneInsert(self):
993
def testSeparateInserts(self):
994
self.doMerge([b'aaa', b'bbb', b'ccc'],
995
[b'aaa', b'xxx', b'bbb', b'ccc'],
996
[b'aaa', b'bbb', b'yyy', b'ccc'],
997
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
999
def testSameInsert(self):
1000
self.doMerge([b'aaa', b'bbb', b'ccc'],
1001
[b'aaa', b'xxx', b'bbb', b'ccc'],
1002
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'],
1003
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
1004
overlappedInsertExpected = [b'aaa', b'xxx', b'yyy', b'bbb']
1006
def testOverlappedInsert(self):
1007
self.doMerge([b'aaa', b'bbb'],
1008
[b'aaa', b'xxx', b'yyy', b'bbb'],
1009
[b'aaa', b'xxx', b'bbb'], self.overlappedInsertExpected)
1011
# really it ought to reduce this to
1012
# [b'aaa', b'xxx', b'yyy', b'bbb']
1014
def testClashReplace(self):
1015
self.doMerge([b'aaa'],
1018
[b'<<<<<<< ', b'xxx', b'=======', b'yyy', b'zzz',
1021
def testNonClashInsert1(self):
1022
self.doMerge([b'aaa'],
1025
[b'<<<<<<< ', b'xxx', b'aaa', b'=======', b'yyy', b'zzz',
1028
def testNonClashInsert2(self):
1029
self.doMerge([b'aaa'],
1034
def testDeleteAndModify(self):
1035
"""Clashing delete and modification.
1037
If one side modifies a region and the other deletes it then
1038
there should be a conflict with one side blank.
1041
#######################################
1042
# skippd, not working yet
1045
self.doMerge([b'aaa', b'bbb', b'ccc'],
1046
[b'aaa', b'ddd', b'ccc'],
1048
[b'<<<<<<<< ', b'aaa', b'=======', b'>>>>>>> ', b'ccc'])
1050
def _test_merge_from_strings(self, base, a, b, expected):
1052
w.add_lines(b'text0', [], base.splitlines(True))
1053
w.add_lines(b'text1', [b'text0'], a.splitlines(True))
1054
w.add_lines(b'text2', [b'text0'], b.splitlines(True))
1055
self.log('merge plan:')
1056
p = list(w.plan_merge(b'text1', b'text2'))
1057
for state, line in p:
1059
self.log('%12s | %s' % (state, line[:-1]))
1060
self.log('merge result:')
1061
result_text = b''.join(w.weave_merge(p))
1062
self.log(result_text)
1063
self.assertEqualDiff(result_text, expected)
1065
def test_weave_merge_conflicts(self):
1066
# does weave merge properly handle plans that end with unchanged?
1067
result = b''.join(self.get_file().weave_merge([('new-a', b'hello\n')]))
1068
self.assertEqual(result, b'hello\n')
1070
def test_deletion_extended(self):
1071
"""One side deletes, the other deletes more.
1092
self._test_merge_from_strings(base, a, b, result)
1094
def test_deletion_overlap(self):
1095
"""Delete overlapping regions with no other conflict.
1097
Arguably it'd be better to treat these as agreement, rather than
1098
conflict, but for now conflict is safer.
1126
self._test_merge_from_strings(base, a, b, result)
1128
def test_agreement_deletion(self):
1129
"""Agree to delete some lines, without conflicts."""
1151
self._test_merge_from_strings(base, a, b, result)
1153
def test_sync_on_deletion(self):
1154
"""Specific case of merge where we can synchronize incorrectly.
1156
A previous version of the weave merge concluded that the two versions
1157
agreed on deleting line 2, and this could be a synchronization point.
1158
Line 1 was then considered in isolation, and thought to be deleted on
1161
It's better to consider the whole thing as a disagreement region.
1172
a's replacement line 2
1185
a's replacement line 2
1192
self._test_merge_from_strings(base, a, b, result)
1195
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1197
def get_file(self, name='foo'):
1198
return WeaveFile(name, self.get_transport(),
1201
def log_contents(self, w):
1202
self.log('weave is:')
1204
write_weave(w, tmpf)
1205
self.log(tmpf.getvalue())
1207
overlappedInsertExpected = [b'aaa', b'<<<<<<< ', b'xxx', b'yyy', b'=======',
1208
b'xxx', b'>>>>>>> ', b'bbb']
1211
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1213
def test_select_adaptor(self):
1214
"""Test expected adapters exist."""
1215
# One scenario for each lookup combination we expect to use.
1216
# Each is source_kind, requested_kind, adapter class
1218
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1219
('knit-delta-gz', 'lines', _mod_knit.DeltaPlainToFullText),
1220
('knit-delta-gz', 'chunked', _mod_knit.DeltaPlainToFullText),
1221
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1222
('knit-ft-gz', 'lines', _mod_knit.FTPlainToFullText),
1223
('knit-ft-gz', 'chunked', _mod_knit.FTPlainToFullText),
1224
('knit-annotated-delta-gz', 'knit-delta-gz',
1225
_mod_knit.DeltaAnnotatedToUnannotated),
1226
('knit-annotated-delta-gz', 'fulltext',
1227
_mod_knit.DeltaAnnotatedToFullText),
1228
('knit-annotated-ft-gz', 'knit-ft-gz',
1229
_mod_knit.FTAnnotatedToUnannotated),
1230
('knit-annotated-ft-gz', 'fulltext',
1231
_mod_knit.FTAnnotatedToFullText),
1232
('knit-annotated-ft-gz', 'lines',
1233
_mod_knit.FTAnnotatedToFullText),
1234
('knit-annotated-ft-gz', 'chunked',
1235
_mod_knit.FTAnnotatedToFullText),
1237
for source, requested, klass in scenarios:
1238
adapter_factory = versionedfile.adapter_registry.get(
1239
(source, requested))
1240
adapter = adapter_factory(None)
1241
self.assertIsInstance(adapter, klass)
1243
def get_knit(self, annotated=True):
1244
mapper = ConstantMapper('knit')
1245
transport = self.get_transport()
1246
return make_file_factory(annotated, mapper)(transport)
1248
def helpGetBytes(self, f, ft_name, ft_adapter, delta_name, delta_adapter):
1249
"""Grab the interested adapted texts for tests."""
1250
# origin is a fulltext
1251
entries = f.get_record_stream([(b'origin',)], 'unordered', False)
1252
base = next(entries)
1253
ft_data = ft_adapter.get_bytes(base, ft_name)
1254
# merged is both a delta and multiple parents.
1255
entries = f.get_record_stream([(b'merged',)], 'unordered', False)
1256
merged = next(entries)
1257
delta_data = delta_adapter.get_bytes(merged, delta_name)
1258
return ft_data, delta_data
1260
def test_deannotation_noeol(self):
1261
"""Test converting annotated knits to unannotated knits."""
1262
# we need a full text, and a delta
1264
get_diamond_files(f, 1, trailing_eol=False)
1265
ft_data, delta_data = self.helpGetBytes(
1266
f, 'knit-ft-gz', _mod_knit.FTAnnotatedToUnannotated(None),
1267
'knit-delta-gz', _mod_knit.DeltaAnnotatedToUnannotated(None))
1269
b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1272
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1274
b'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1275
b'1,2,3\nleft\nright\nmerged\nend merged\n',
1276
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1278
def test_deannotation(self):
1279
"""Test converting annotated knits to unannotated knits."""
1280
# we need a full text, and a delta
1282
get_diamond_files(f, 1)
1283
ft_data, delta_data = self.helpGetBytes(
1284
f, 'knit-ft-gz', _mod_knit.FTAnnotatedToUnannotated(None),
1285
'knit-delta-gz', _mod_knit.DeltaAnnotatedToUnannotated(None))
1287
b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1290
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1292
b'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1293
b'2,2,2\nright\nmerged\nend merged\n',
1294
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1296
def test_annotated_to_fulltext_no_eol(self):
1297
"""Test adapting annotated knits to full texts (for -> weaves)."""
1298
# we need a full text, and a delta
1300
get_diamond_files(f, 1, trailing_eol=False)
1301
# Reconstructing a full text requires a backing versioned file, and it
1302
# must have the base lines requested from it.
1303
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1304
ft_data, delta_data = self.helpGetBytes(
1305
f, 'fulltext', _mod_knit.FTAnnotatedToFullText(None),
1306
'fulltext', _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1307
self.assertEqual(b'origin', ft_data)
1308
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1309
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1310
True)], logged_vf.calls)
1312
def test_annotated_to_fulltext(self):
1313
"""Test adapting annotated knits to full texts (for -> weaves)."""
1314
# we need a full text, and a delta
1316
get_diamond_files(f, 1)
1317
# Reconstructing a full text requires a backing versioned file, and it
1318
# must have the base lines requested from it.
1319
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1320
ft_data, delta_data = self.helpGetBytes(
1321
f, 'fulltext', _mod_knit.FTAnnotatedToFullText(None),
1322
'fulltext', _mod_knit.DeltaAnnotatedToFullText(logged_vf))
1323
self.assertEqual(b'origin\n', ft_data)
1324
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1325
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1326
True)], logged_vf.calls)
1328
def test_unannotated_to_fulltext(self):
1329
"""Test adapting unannotated knits to full texts.
1331
This is used for -> weaves, and for -> annotated knits.
1333
# we need a full text, and a delta
1334
f = self.get_knit(annotated=False)
1335
get_diamond_files(f, 1)
1336
# Reconstructing a full text requires a backing versioned file, and it
1337
# must have the base lines requested from it.
1338
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1339
ft_data, delta_data = self.helpGetBytes(
1340
f, 'fulltext', _mod_knit.FTPlainToFullText(None),
1341
'fulltext', _mod_knit.DeltaPlainToFullText(logged_vf))
1342
self.assertEqual(b'origin\n', ft_data)
1343
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1344
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1345
True)], logged_vf.calls)
1347
def test_unannotated_to_fulltext_no_eol(self):
1348
"""Test adapting unannotated knits to full texts.
1350
This is used for -> weaves, and for -> annotated knits.
1352
# we need a full text, and a delta
1353
f = self.get_knit(annotated=False)
1354
get_diamond_files(f, 1, trailing_eol=False)
1355
# Reconstructing a full text requires a backing versioned file, and it
1356
# must have the base lines requested from it.
1357
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1358
ft_data, delta_data = self.helpGetBytes(
1359
f, 'fulltext', _mod_knit.FTPlainToFullText(None),
1360
'fulltext', _mod_knit.DeltaPlainToFullText(logged_vf))
1361
self.assertEqual(b'origin', ft_data)
1362
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1363
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1364
True)], logged_vf.calls)
1367
class TestKeyMapper(TestCaseWithMemoryTransport):
1368
"""Tests for various key mapping logic."""
1370
def test_identity_mapper(self):
1371
mapper = versionedfile.ConstantMapper("inventory")
1372
self.assertEqual("inventory", mapper.map((b'foo@ar',)))
1373
self.assertEqual("inventory", mapper.map((b'quux',)))
1375
def test_prefix_mapper(self):
1377
mapper = versionedfile.PrefixMapper()
1378
self.assertEqual("file-id", mapper.map((b"file-id", b"revision-id")))
1379
self.assertEqual("new-id", mapper.map((b"new-id", b"revision-id")))
1380
self.assertEqual((b'file-id',), mapper.unmap("file-id"))
1381
self.assertEqual((b'new-id',), mapper.unmap("new-id"))
1383
def test_hash_prefix_mapper(self):
1384
#format6: hash + plain
1385
mapper = versionedfile.HashPrefixMapper()
1387
"9b/file-id", mapper.map((b"file-id", b"revision-id")))
1388
self.assertEqual("45/new-id", mapper.map((b"new-id", b"revision-id")))
1389
self.assertEqual((b'file-id',), mapper.unmap("9b/file-id"))
1390
self.assertEqual((b'new-id',), mapper.unmap("45/new-id"))
1392
def test_hash_escaped_mapper(self):
1393
#knit1: hash + escaped
1394
mapper = versionedfile.HashEscapedPrefixMapper()
1395
self.assertEqual("88/%2520", mapper.map((b" ", b"revision-id")))
1396
self.assertEqual("ed/fil%2545-%2549d", mapper.map((b"filE-Id",
1398
self.assertEqual("88/ne%2557-%2549d", mapper.map((b"neW-Id",
1400
self.assertEqual((b'filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1401
self.assertEqual((b'neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1404
class TestVersionedFiles(TestCaseWithMemoryTransport):
1405
"""Tests for the multiple-file variant of VersionedFile."""
1407
# We want to be sure of behaviour for:
1408
# weaves prefix layout (weave texts)
1409
# individually named weaves (weave inventories)
1410
# annotated knits - prefix|hash|hash-escape layout, we test the third only
1411
# as it is the most complex mapper.
1412
# individually named knits
1413
# individual no-graph knits in packs (signatures)
1414
# individual graph knits in packs (inventories)
1415
# individual graph nocompression knits in packs (revisions)
1416
# plain text knits in packs (texts)
1417
len_one_scenarios = [
1420
'factory': make_versioned_files_factory(WeaveFile,
1421
ConstantMapper('inventory')),
1424
'support_partial_insertion': False,
1428
'factory': make_file_factory(False, ConstantMapper('revisions')),
1431
'support_partial_insertion': False,
1433
('named-nograph-nodelta-knit-pack', {
1434
'cleanup': cleanup_pack_knit,
1435
'factory': make_pack_factory(False, False, 1),
1438
'support_partial_insertion': False,
1440
('named-graph-knit-pack', {
1441
'cleanup': cleanup_pack_knit,
1442
'factory': make_pack_factory(True, True, 1),
1445
'support_partial_insertion': True,
1447
('named-graph-nodelta-knit-pack', {
1448
'cleanup': cleanup_pack_knit,
1449
'factory': make_pack_factory(True, False, 1),
1452
'support_partial_insertion': False,
1454
('groupcompress-nograph', {
1455
'cleanup': groupcompress.cleanup_pack_group,
1456
'factory': groupcompress.make_pack_factory(False, False, 1),
1459
'support_partial_insertion': False,
1462
len_two_scenarios = [
1465
'factory': make_versioned_files_factory(WeaveFile,
1469
'support_partial_insertion': False,
1471
('annotated-knit-escape', {
1473
'factory': make_file_factory(True, HashEscapedPrefixMapper()),
1476
'support_partial_insertion': False,
1478
('plain-knit-pack', {
1479
'cleanup': cleanup_pack_knit,
1480
'factory': make_pack_factory(True, True, 2),
1483
'support_partial_insertion': True,
1486
'cleanup': groupcompress.cleanup_pack_group,
1487
'factory': groupcompress.make_pack_factory(True, False, 1),
1490
'support_partial_insertion': False,
1494
scenarios = len_one_scenarios + len_two_scenarios
1496
def get_versionedfiles(self, relpath='files'):
1497
transport = self.get_transport(relpath)
1499
transport.mkdir('.')
1500
files = self.factory(transport)
1501
if self.cleanup is not None:
1502
self.addCleanup(self.cleanup, files)
1505
def get_simple_key(self, suffix):
1506
"""Return a key for the object under test."""
1507
if self.key_length == 1:
1510
return (b'FileA',) + (suffix,)
1512
def test_add_fallback_implies_without_fallbacks(self):
1513
f = self.get_versionedfiles('files')
1514
if getattr(f, 'add_fallback_versioned_files', None) is None:
1515
raise TestNotApplicable("%s doesn't support fallbacks"
1516
% (f.__class__.__name__,))
1517
g = self.get_versionedfiles('fallback')
1518
key_a = self.get_simple_key(b'a')
1519
g.add_lines(key_a, [], [b'\n'])
1520
f.add_fallback_versioned_files(g)
1521
self.assertTrue(key_a in f.get_parent_map([key_a]))
1523
key_a in f.without_fallbacks().get_parent_map([key_a]))
1525
def test_add_lines(self):
1526
f = self.get_versionedfiles()
1527
key0 = self.get_simple_key(b'r0')
1528
key1 = self.get_simple_key(b'r1')
1529
key2 = self.get_simple_key(b'r2')
1530
keyf = self.get_simple_key(b'foo')
1531
f.add_lines(key0, [], [b'a\n', b'b\n'])
1533
f.add_lines(key1, [key0], [b'b\n', b'c\n'])
1535
f.add_lines(key1, [], [b'b\n', b'c\n'])
1537
self.assertTrue(key0 in keys)
1538
self.assertTrue(key1 in keys)
1540
for record in f.get_record_stream([key0, key1], 'unordered', True):
1541
records.append((record.key, record.get_bytes_as('fulltext')))
1543
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1545
def test_add_chunks(self):
1546
f = self.get_versionedfiles()
1547
key0 = self.get_simple_key(b'r0')
1548
key1 = self.get_simple_key(b'r1')
1549
key2 = self.get_simple_key(b'r2')
1550
keyf = self.get_simple_key(b'foo')
1551
def add_chunks(key, parents, chunks):
1552
factory = ChunkedContentFactory(
1553
key, parents, osutils.sha_strings(chunks), chunks)
1554
return f.add_content(factory)
1556
add_chunks(key0, [], [b'a', b'\nb\n'])
1558
add_chunks(key1, [key0], [b'b', b'\n', b'c\n'])
1560
add_chunks(key1, [], [b'b\n', b'c\n'])
1562
self.assertIn(key0, keys)
1563
self.assertIn(key1, keys)
1565
for record in f.get_record_stream([key0, key1], 'unordered', True):
1566
records.append((record.key, record.get_bytes_as('fulltext')))
1568
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1570
def test_annotate(self):
1571
files = self.get_versionedfiles()
1572
self.get_diamond_files(files)
1573
if self.key_length == 1:
1576
prefix = (b'FileA',)
1577
# introduced full text
1578
origins = files.annotate(prefix + (b'origin',))
1580
(prefix + (b'origin',), b'origin\n')],
1583
origins = files.annotate(prefix + (b'base',))
1585
(prefix + (b'base',), b'base\n')],
1588
origins = files.annotate(prefix + (b'merged',))
1591
(prefix + (b'base',), b'base\n'),
1592
(prefix + (b'left',), b'left\n'),
1593
(prefix + (b'right',), b'right\n'),
1594
(prefix + (b'merged',), b'merged\n')
1598
# Without a graph everything is new.
1600
(prefix + (b'merged',), b'base\n'),
1601
(prefix + (b'merged',), b'left\n'),
1602
(prefix + (b'merged',), b'right\n'),
1603
(prefix + (b'merged',), b'merged\n')
1606
self.assertRaises(RevisionNotPresent,
1607
files.annotate, prefix + ('missing-key',))
1609
def test_check_no_parameters(self):
1610
files = self.get_versionedfiles()
1612
def test_check_progressbar_parameter(self):
1613
"""A progress bar can be supplied because check can be a generator."""
1614
pb = ui.ui_factory.nested_progress_bar()
1615
self.addCleanup(pb.finished)
1616
files = self.get_versionedfiles()
1617
files.check(progress_bar=pb)
1619
def test_check_with_keys_becomes_generator(self):
1620
files = self.get_versionedfiles()
1621
self.get_diamond_files(files)
1623
entries = files.check(keys=keys)
1625
# Texts output should be fulltexts.
1626
self.capture_stream(files, entries, seen.add,
1627
files.get_parent_map(keys), require_fulltext=True)
1628
# All texts should be output.
1629
self.assertEqual(set(keys), seen)
1631
def test_clear_cache(self):
1632
files = self.get_versionedfiles()
1635
def test_construct(self):
1636
"""Each parameterised test can be constructed on a transport."""
1637
files = self.get_versionedfiles()
1639
def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1641
return get_diamond_files(files, self.key_length,
1642
trailing_eol=trailing_eol, nograph=not self.graph,
1643
left_only=left_only, nokeys=nokeys)
1645
def _add_content_nostoresha(self, add_lines):
1646
"""When nostore_sha is supplied using old content raises."""
1647
vf = self.get_versionedfiles()
1648
empty_text = (b'a', [])
1649
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
1650
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
1652
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1654
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1657
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1660
# we now have a copy of all the lines in the vf.
1661
for sha, (version, lines) in zip(
1662
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1663
new_key = self.get_simple_key(version + b"2")
1664
self.assertRaises(errors.ExistingContent,
1665
vf.add_lines, new_key, [], lines,
1667
self.assertRaises(errors.ExistingContent,
1668
vf.add_lines, new_key, [], lines,
1670
# and no new version should have been added.
1671
record = next(vf.get_record_stream([new_key], 'unordered', True))
1672
self.assertEqual('absent', record.storage_kind)
1674
def test_add_lines_nostoresha(self):
1675
self._add_content_nostoresha(add_lines=True)
1677
def test_add_lines_return(self):
1678
files = self.get_versionedfiles()
1679
# save code by using the stock data insertion helper.
1680
adds = self.get_diamond_files(files)
1682
# We can only validate the first 2 elements returned from add_lines.
1684
self.assertEqual(3, len(add))
1685
results.append(add[:2])
1686
if self.key_length == 1:
1688
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1689
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1690
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1691
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1692
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1694
elif self.key_length == 2:
1696
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1697
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1698
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1699
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1700
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1701
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1702
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1703
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1704
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1705
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1708
def test_add_lines_no_key_generates_chk_key(self):
1709
files = self.get_versionedfiles()
1710
# save code by using the stock data insertion helper.
1711
adds = self.get_diamond_files(files, nokeys=True)
1713
# We can only validate the first 2 elements returned from add_lines.
1715
self.assertEqual(3, len(add))
1716
results.append(add[:2])
1717
if self.key_length == 1:
1719
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1720
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1721
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1722
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1723
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1725
# Check the added items got CHK keys.
1727
(b'sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1728
(b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1729
(b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1730
(b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1731
(b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1734
elif self.key_length == 2:
1736
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1737
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1738
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1739
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1740
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1741
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1742
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1743
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1744
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1745
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1747
# Check the added items got CHK keys.
1749
(b'FileA', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1750
(b'FileA', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1751
(b'FileA', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1752
(b'FileA', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1753
(b'FileA', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1754
(b'FileB', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1755
(b'FileB', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1756
(b'FileB', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1757
(b'FileB', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1758
(b'FileB', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1762
def test_empty_lines(self):
1763
"""Empty files can be stored."""
1764
f = self.get_versionedfiles()
1765
key_a = self.get_simple_key(b'a')
1766
f.add_lines(key_a, [], [])
1767
self.assertEqual(b'',
1768
next(f.get_record_stream([key_a], 'unordered', True
1769
)).get_bytes_as('fulltext'))
1770
key_b = self.get_simple_key(b'b')
1771
f.add_lines(key_b, self.get_parents([key_a]), [])
1772
self.assertEqual(b'',
1773
next(f.get_record_stream([key_b], 'unordered', True
1774
)).get_bytes_as('fulltext'))
1776
def test_newline_only(self):
1777
f = self.get_versionedfiles()
1778
key_a = self.get_simple_key(b'a')
1779
f.add_lines(key_a, [], [b'\n'])
1780
self.assertEqual(b'\n',
1781
next(f.get_record_stream([key_a], 'unordered', True
1782
)).get_bytes_as('fulltext'))
1783
key_b = self.get_simple_key(b'b')
1784
f.add_lines(key_b, self.get_parents([key_a]), [b'\n'])
1785
self.assertEqual(b'\n',
1786
next(f.get_record_stream([key_b], 'unordered', True
1787
)).get_bytes_as('fulltext'))
1789
def test_get_known_graph_ancestry(self):
1790
f = self.get_versionedfiles()
1792
raise TestNotApplicable('ancestry info only relevant with graph.')
1793
key_a = self.get_simple_key(b'a')
1794
key_b = self.get_simple_key(b'b')
1795
key_c = self.get_simple_key(b'c')
1801
f.add_lines(key_a, [], [b'\n'])
1802
f.add_lines(key_b, [key_a], [b'\n'])
1803
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1804
kg = f.get_known_graph_ancestry([key_c])
1805
self.assertIsInstance(kg, _mod_graph.KnownGraph)
1806
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1808
def test_known_graph_with_fallbacks(self):
1809
f = self.get_versionedfiles('files')
1811
raise TestNotApplicable('ancestry info only relevant with graph.')
1812
if getattr(f, 'add_fallback_versioned_files', None) is None:
1813
raise TestNotApplicable("%s doesn't support fallbacks"
1814
% (f.__class__.__name__,))
1815
key_a = self.get_simple_key(b'a')
1816
key_b = self.get_simple_key(b'b')
1817
key_c = self.get_simple_key(b'c')
1818
# A only in fallback
1823
g = self.get_versionedfiles('fallback')
1824
g.add_lines(key_a, [], [b'\n'])
1825
f.add_fallback_versioned_files(g)
1826
f.add_lines(key_b, [key_a], [b'\n'])
1827
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1828
kg = f.get_known_graph_ancestry([key_c])
1829
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1831
def test_get_record_stream_empty(self):
1832
"""An empty stream can be requested without error."""
1833
f = self.get_versionedfiles()
1834
entries = f.get_record_stream([], 'unordered', False)
1835
self.assertEqual([], list(entries))
1837
def assertValidStorageKind(self, storage_kind):
1838
"""Assert that storage_kind is a valid storage_kind."""
1839
self.assertSubset([storage_kind],
1840
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1841
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1842
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1844
'knit-delta-closure', 'knit-delta-closure-ref',
1845
'groupcompress-block', 'groupcompress-block-ref'])
1847
def capture_stream(self, f, entries, on_seen, parents,
1848
require_fulltext=False):
1849
"""Capture a stream for testing."""
1850
for factory in entries:
1851
on_seen(factory.key)
1852
self.assertValidStorageKind(factory.storage_kind)
1853
if factory.sha1 is not None:
1854
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1856
self.assertEqual(parents[factory.key], factory.parents)
1857
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1859
if require_fulltext:
1860
factory.get_bytes_as('fulltext')
1862
def test_get_record_stream_interface(self):
1863
"""each item in a stream has to provide a regular interface."""
1864
files = self.get_versionedfiles()
1865
self.get_diamond_files(files)
1866
keys, _ = self.get_keys_and_sort_order()
1867
parent_map = files.get_parent_map(keys)
1868
entries = files.get_record_stream(keys, 'unordered', False)
1870
self.capture_stream(files, entries, seen.add, parent_map)
1871
self.assertEqual(set(keys), seen)
1873
def get_keys_and_sort_order(self):
1874
"""Get diamond test keys list, and their sort ordering."""
1875
if self.key_length == 1:
1876
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1877
sort_order = {(b'merged',): 2, (b'left',): 1,
1878
(b'right',): 1, (b'base',): 0}
1881
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1882
(b'FileA', b'base'),
1883
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1884
(b'FileB', b'base'),
1887
(b'FileA', b'merged'): 2, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1888
(b'FileA', b'base'): 0,
1889
(b'FileB', b'merged'): 2, (b'FileB', b'left'): 1, (b'FileB', b'right'): 1,
1890
(b'FileB', b'base'): 0,
1892
return keys, sort_order
1894
def get_keys_and_groupcompress_sort_order(self):
1895
"""Get diamond test keys list, and their groupcompress sort ordering."""
1896
if self.key_length == 1:
1897
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1898
sort_order = {(b'merged',): 0, (b'left',): 1,
1899
(b'right',): 1, (b'base',): 2}
1902
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1903
(b'FileA', b'base'),
1904
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1905
(b'FileB', b'base'),
1908
(b'FileA', b'merged'): 0, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1909
(b'FileA', b'base'): 2,
1910
(b'FileB', b'merged'): 3, (b'FileB', b'left'): 4, (b'FileB', b'right'): 4,
1911
(b'FileB', b'base'): 5,
1913
return keys, sort_order
1915
def test_get_record_stream_interface_ordered(self):
1916
"""each item in a stream has to provide a regular interface."""
1917
files = self.get_versionedfiles()
1918
self.get_diamond_files(files)
1919
keys, sort_order = self.get_keys_and_sort_order()
1920
parent_map = files.get_parent_map(keys)
1921
entries = files.get_record_stream(keys, 'topological', False)
1923
self.capture_stream(files, entries, seen.append, parent_map)
1924
self.assertStreamOrder(sort_order, seen, keys)
1926
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1927
"""each item must be accessible as a fulltext."""
1928
files = self.get_versionedfiles()
1929
self.get_diamond_files(files)
1930
keys, sort_order = self.get_keys_and_sort_order()
1931
parent_map = files.get_parent_map(keys)
1932
entries = files.get_record_stream(keys, 'topological', True)
1934
for factory in entries:
1935
seen.append(factory.key)
1936
self.assertValidStorageKind(factory.storage_kind)
1937
self.assertSubset([factory.sha1],
1938
[None, files.get_sha1s([factory.key])[factory.key]])
1939
self.assertEqual(parent_map[factory.key], factory.parents)
1940
# self.assertEqual(files.get_text(factory.key),
1941
ft_bytes = factory.get_bytes_as('fulltext')
1942
self.assertIsInstance(ft_bytes, bytes)
1943
chunked_bytes = factory.get_bytes_as('chunked')
1944
self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1945
chunked_bytes = factory.iter_bytes_as('chunked')
1946
self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1948
self.assertStreamOrder(sort_order, seen, keys)
1950
def test_get_record_stream_interface_groupcompress(self):
1951
"""each item in a stream has to provide a regular interface."""
1952
files = self.get_versionedfiles()
1953
self.get_diamond_files(files)
1954
keys, sort_order = self.get_keys_and_groupcompress_sort_order()
1955
parent_map = files.get_parent_map(keys)
1956
entries = files.get_record_stream(keys, 'groupcompress', False)
1958
self.capture_stream(files, entries, seen.append, parent_map)
1959
self.assertStreamOrder(sort_order, seen, keys)
1961
def assertStreamOrder(self, sort_order, seen, keys):
1962
self.assertEqual(len(set(seen)), len(keys))
1963
if self.key_length == 1:
1966
lows = {(b'FileA',): 0, (b'FileB',): 0}
1968
self.assertEqual(set(keys), set(seen))
1971
sort_pos = sort_order[key]
1972
self.assertTrue(sort_pos >= lows[key[:-1]],
1973
"Out of order in sorted stream: %r, %r" % (key, seen))
1974
lows[key[:-1]] = sort_pos
1976
def test_get_record_stream_unknown_storage_kind_raises(self):
1977
"""Asking for a storage kind that the stream cannot supply raises."""
1978
files = self.get_versionedfiles()
1979
self.get_diamond_files(files)
1980
if self.key_length == 1:
1981
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1984
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1985
(b'FileA', b'base'),
1986
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1987
(b'FileB', b'base'),
1989
parent_map = files.get_parent_map(keys)
1990
entries = files.get_record_stream(keys, 'unordered', False)
1991
# We track the contents because we should be able to try, fail a
1992
# particular kind and then ask for one that works and continue.
1994
for factory in entries:
1995
seen.add(factory.key)
1996
self.assertValidStorageKind(factory.storage_kind)
1997
if factory.sha1 is not None:
1998
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
2000
self.assertEqual(parent_map[factory.key], factory.parents)
2001
# currently no stream emits mpdiff
2002
self.assertRaises(errors.UnavailableRepresentation,
2003
factory.get_bytes_as, 'mpdiff')
2004
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2006
self.assertEqual(set(keys), seen)
2008
def test_get_record_stream_missing_records_are_absent(self):
2009
files = self.get_versionedfiles()
2010
self.get_diamond_files(files)
2011
if self.key_length == 1:
2012
keys = [(b'merged',), (b'left',), (b'right',),
2013
(b'absent',), (b'base',)]
2016
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
2017
(b'FileA', b'absent'), (b'FileA', b'base'),
2018
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
2019
(b'FileB', b'absent'), (b'FileB', b'base'),
2020
(b'absent', b'absent'),
2022
parent_map = files.get_parent_map(keys)
2023
entries = files.get_record_stream(keys, 'unordered', False)
2024
self.assertAbsentRecord(files, keys, parent_map, entries)
2025
entries = files.get_record_stream(keys, 'topological', False)
2026
self.assertAbsentRecord(files, keys, parent_map, entries)
2028
def assertRecordHasContent(self, record, bytes):
2029
"""Assert that record has the bytes bytes."""
2030
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
2031
self.assertEqual(bytes, b''.join(record.get_bytes_as('chunked')))
2033
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
2034
files = self.get_versionedfiles()
2035
key = self.get_simple_key(b'foo')
2036
files.add_lines(key, (), [b'my text\n', b'content'])
2037
stream = files.get_record_stream([key], 'unordered', False)
2038
record = next(stream)
2039
if record.storage_kind in ('chunked', 'fulltext'):
2040
# chunked and fulltext representations are for direct use not wire
2041
# serialisation: check they are able to be used directly. To send
2042
# such records over the wire translation will be needed.
2043
self.assertRecordHasContent(record, b"my text\ncontent")
2045
bytes = [record.get_bytes_as(record.storage_kind)]
2046
network_stream = versionedfile.NetworkRecordStream(bytes).read()
2047
source_record = record
2049
for record in network_stream:
2050
records.append(record)
2051
self.assertEqual(source_record.storage_kind,
2052
record.storage_kind)
2053
self.assertEqual(source_record.parents, record.parents)
2055
source_record.get_bytes_as(source_record.storage_kind),
2056
record.get_bytes_as(record.storage_kind))
2057
self.assertEqual(1, len(records))
2059
def assertStreamMetaEqual(self, records, expected, stream):
2060
"""Assert that streams expected and stream have the same records.
2062
:param records: A list to collect the seen records.
2063
:return: A generator of the records in stream.
2065
# We make assertions during copying to catch things early for easier
2066
# debugging. This must use the iterating zip() from the future.
2067
for record, ref_record in zip(stream, expected):
2068
records.append(record)
2069
self.assertEqual(ref_record.key, record.key)
2070
self.assertEqual(ref_record.storage_kind, record.storage_kind)
2071
self.assertEqual(ref_record.parents, record.parents)
2074
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
2076
"""Convert a stream to a bytes iterator.
2078
:param skipped_records: A list with one element to increment when a
2080
:param full_texts: A dict from key->fulltext representation, for
2081
checking chunked or fulltext stored records.
2082
:param stream: A record_stream.
2083
:return: An iterator over the bytes of each record.
2085
for record in stream:
2086
if record.storage_kind in ('chunked', 'fulltext'):
2087
skipped_records[0] += 1
2088
# check the content is correct for direct use.
2089
self.assertRecordHasContent(record, full_texts[record.key])
2091
yield record.get_bytes_as(record.storage_kind)
2093
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
2094
files = self.get_versionedfiles()
2095
target_files = self.get_versionedfiles('target')
2096
key = self.get_simple_key(b'ft')
2097
key_delta = self.get_simple_key(b'delta')
2098
files.add_lines(key, (), [b'my text\n', b'content'])
2100
delta_parents = (key,)
2103
files.add_lines(key_delta, delta_parents, [
2104
b'different\n', b'content\n'])
2105
local = files.get_record_stream([key, key_delta], 'unordered', False)
2106
ref = files.get_record_stream([key, key_delta], 'unordered', False)
2107
skipped_records = [0]
2109
key: b"my text\ncontent",
2110
key_delta: b"different\ncontent\n",
2112
byte_stream = self.stream_to_bytes_or_skip_counter(
2113
skipped_records, full_texts, local)
2114
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2116
# insert the stream from the network into a versioned files object so we can
2117
# check the content was carried across correctly without doing delta
2119
target_files.insert_record_stream(
2120
self.assertStreamMetaEqual(records, ref, network_stream))
2121
# No duplicates on the wire thank you!
2122
self.assertEqual(2, len(records) + skipped_records[0])
2124
# if any content was copied it all must have all been.
2125
self.assertIdenticalVersionedFile(files, target_files)
2127
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
2128
# copy a delta over the wire
2129
files = self.get_versionedfiles()
2130
target_files = self.get_versionedfiles('target')
2131
key = self.get_simple_key(b'ft')
2132
key_delta = self.get_simple_key(b'delta')
2133
files.add_lines(key, (), [b'my text\n', b'content'])
2135
delta_parents = (key,)
2138
files.add_lines(key_delta, delta_parents, [
2139
b'different\n', b'content\n'])
2140
# Copy the basis text across so we can reconstruct the delta during
2141
# insertion into target.
2142
target_files.insert_record_stream(files.get_record_stream([key],
2143
'unordered', False))
2144
local = files.get_record_stream([key_delta], 'unordered', False)
2145
ref = files.get_record_stream([key_delta], 'unordered', False)
2146
skipped_records = [0]
2148
key_delta: b"different\ncontent\n",
2150
byte_stream = self.stream_to_bytes_or_skip_counter(
2151
skipped_records, full_texts, local)
2152
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2154
# insert the stream from the network into a versioned files object so we can
2155
# check the content was carried across correctly without doing delta
2156
# inspection during check_stream.
2157
target_files.insert_record_stream(
2158
self.assertStreamMetaEqual(records, ref, network_stream))
2159
# No duplicates on the wire thank you!
2160
self.assertEqual(1, len(records) + skipped_records[0])
2162
# if any content was copied it all must have all been
2163
self.assertIdenticalVersionedFile(files, target_files)
2165
def test_get_record_stream_wire_ready_delta_closure_included(self):
2166
# copy a delta over the wire with the ability to get its full text.
2167
files = self.get_versionedfiles()
2168
key = self.get_simple_key(b'ft')
2169
key_delta = self.get_simple_key(b'delta')
2170
files.add_lines(key, (), [b'my text\n', b'content'])
2172
delta_parents = (key,)
2175
files.add_lines(key_delta, delta_parents, [
2176
b'different\n', b'content\n'])
2177
local = files.get_record_stream([key_delta], 'unordered', True)
2178
ref = files.get_record_stream([key_delta], 'unordered', True)
2179
skipped_records = [0]
2181
key_delta: b"different\ncontent\n",
2183
byte_stream = self.stream_to_bytes_or_skip_counter(
2184
skipped_records, full_texts, local)
2185
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2187
# insert the stream from the network into a versioned files object so we can
2188
# check the content was carried across correctly without doing delta
2189
# inspection during check_stream.
2190
for record in self.assertStreamMetaEqual(records, ref, network_stream):
2191
# we have to be able to get the full text out:
2192
self.assertRecordHasContent(record, full_texts[record.key])
2193
# No duplicates on the wire thank you!
2194
self.assertEqual(1, len(records) + skipped_records[0])
2196
def assertAbsentRecord(self, files, keys, parents, entries):
2197
"""Helper for test_get_record_stream_missing_records_are_absent."""
2199
for factory in entries:
2200
seen.add(factory.key)
2201
if factory.key[-1] == b'absent':
2202
self.assertEqual('absent', factory.storage_kind)
2203
self.assertEqual(None, factory.sha1)
2204
self.assertEqual(None, factory.parents)
2206
self.assertValidStorageKind(factory.storage_kind)
2207
if factory.sha1 is not None:
2208
sha1 = files.get_sha1s([factory.key])[factory.key]
2209
self.assertEqual(sha1, factory.sha1)
2210
self.assertEqual(parents[factory.key], factory.parents)
2211
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2213
self.assertEqual(set(keys), seen)
2215
def test_filter_absent_records(self):
2216
"""Requested missing records can be filter trivially."""
2217
files = self.get_versionedfiles()
2218
self.get_diamond_files(files)
2219
keys, _ = self.get_keys_and_sort_order()
2220
parent_map = files.get_parent_map(keys)
2221
# Add an absent record in the middle of the present keys. (We don't ask
2222
# for just absent keys to ensure that content before and after the
2223
# absent keys is still delivered).
2224
present_keys = list(keys)
2225
if self.key_length == 1:
2226
keys.insert(2, (b'extra',))
2228
keys.insert(2, (b'extra', b'extra'))
2229
entries = files.get_record_stream(keys, 'unordered', False)
2231
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2233
self.assertEqual(set(present_keys), seen)
2235
def get_mapper(self):
2236
"""Get a mapper suitable for the key length of the test interface."""
2237
if self.key_length == 1:
2238
return ConstantMapper('source')
2240
return HashEscapedPrefixMapper()
2242
def get_parents(self, parents):
2243
"""Get parents, taking self.graph into consideration."""
2249
def test_get_annotator(self):
2250
files = self.get_versionedfiles()
2251
self.get_diamond_files(files)
2252
origin_key = self.get_simple_key(b'origin')
2253
base_key = self.get_simple_key(b'base')
2254
left_key = self.get_simple_key(b'left')
2255
right_key = self.get_simple_key(b'right')
2256
merged_key = self.get_simple_key(b'merged')
2257
# annotator = files.get_annotator()
2258
# introduced full text
2259
origins, lines = files.get_annotator().annotate(origin_key)
2260
self.assertEqual([(origin_key,)], origins)
2261
self.assertEqual([b'origin\n'], lines)
2263
origins, lines = files.get_annotator().annotate(base_key)
2264
self.assertEqual([(base_key,)], origins)
2266
origins, lines = files.get_annotator().annotate(merged_key)
2275
# Without a graph everything is new.
2282
self.assertRaises(RevisionNotPresent,
2283
files.get_annotator().annotate, self.get_simple_key(b'missing-key'))
2285
def test_get_parent_map(self):
2286
files = self.get_versionedfiles()
2287
if self.key_length == 1:
2289
((b'r0',), self.get_parents(())),
2290
((b'r1',), self.get_parents(((b'r0',),))),
2291
((b'r2',), self.get_parents(())),
2292
((b'r3',), self.get_parents(())),
2293
((b'm',), self.get_parents(((b'r0',), (b'r1',), (b'r2',), (b'r3',)))),
2297
((b'FileA', b'r0'), self.get_parents(())),
2298
((b'FileA', b'r1'), self.get_parents(((b'FileA', b'r0'),))),
2299
((b'FileA', b'r2'), self.get_parents(())),
2300
((b'FileA', b'r3'), self.get_parents(())),
2301
((b'FileA', b'm'), self.get_parents(((b'FileA', b'r0'),
2302
(b'FileA', b'r1'), (b'FileA', b'r2'), (b'FileA', b'r3')))),
2304
for key, parents in parent_details:
2305
files.add_lines(key, parents, [])
2306
# immediately after adding it should be queryable.
2307
self.assertEqual({key: parents}, files.get_parent_map([key]))
2308
# We can ask for an empty set
2309
self.assertEqual({}, files.get_parent_map([]))
2310
# We can ask for many keys
2311
all_parents = dict(parent_details)
2312
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2313
# Absent keys are just not included in the result.
2314
keys = list(all_parents.keys())
2315
if self.key_length == 1:
2316
keys.insert(1, (b'missing',))
2318
keys.insert(1, (b'missing', b'missing'))
2319
# Absent keys are just ignored
2320
self.assertEqual(all_parents, files.get_parent_map(keys))
2322
def test_get_sha1s(self):
2323
files = self.get_versionedfiles()
2324
self.get_diamond_files(files)
2325
if self.key_length == 1:
2326
keys = [(b'base',), (b'origin',), (b'left',),
2327
(b'merged',), (b'right',)]
2329
# ask for shas from different prefixes.
2331
(b'FileA', b'base'), (b'FileB', b'origin'), (b'FileA', b'left'),
2332
(b'FileA', b'merged'), (b'FileB', b'right'),
2335
keys[0]: b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2336
keys[1]: b'00e364d235126be43292ab09cb4686cf703ddc17',
2337
keys[2]: b'a8478686da38e370e32e42e8a0c220e33ee9132f',
2338
keys[3]: b'ed8bce375198ea62444dc71952b22cfc2b09226d',
2339
keys[4]: b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2341
files.get_sha1s(keys))
2343
def test_insert_record_stream_empty(self):
2344
"""Inserting an empty record stream should work."""
2345
files = self.get_versionedfiles()
2346
files.insert_record_stream([])
2348
def assertIdenticalVersionedFile(self, expected, actual):
2349
"""Assert that left and right have the same contents."""
2350
self.assertEqual(set(actual.keys()), set(expected.keys()))
2351
actual_parents = actual.get_parent_map(actual.keys())
2354
actual_parents, expected.get_parent_map(expected.keys()))
2356
for key, parents in actual_parents.items():
2357
self.assertEqual(None, parents)
2358
for key in actual.keys():
2359
actual_text = next(actual.get_record_stream(
2360
[key], 'unordered', True)).get_bytes_as('fulltext')
2361
expected_text = next(expected.get_record_stream(
2362
[key], 'unordered', True)).get_bytes_as('fulltext')
2363
self.assertEqual(actual_text, expected_text)
2365
def test_insert_record_stream_fulltexts(self):
2366
"""Any file should accept a stream of fulltexts."""
2367
files = self.get_versionedfiles()
2368
mapper = self.get_mapper()
2369
source_transport = self.get_transport('source')
2370
source_transport.mkdir('.')
2371
# weaves always output fulltexts.
2372
source = make_versioned_files_factory(WeaveFile, mapper)(
2374
self.get_diamond_files(source, trailing_eol=False)
2375
stream = source.get_record_stream(source.keys(), 'topological',
2377
files.insert_record_stream(stream)
2378
self.assertIdenticalVersionedFile(source, files)
2380
def test_insert_record_stream_fulltexts_noeol(self):
2381
"""Any file should accept a stream of fulltexts."""
2382
files = self.get_versionedfiles()
2383
mapper = self.get_mapper()
2384
source_transport = self.get_transport('source')
2385
source_transport.mkdir('.')
2386
# weaves always output fulltexts.
2387
source = make_versioned_files_factory(WeaveFile, mapper)(
2389
self.get_diamond_files(source, trailing_eol=False)
2390
stream = source.get_record_stream(source.keys(), 'topological',
2392
files.insert_record_stream(stream)
2393
self.assertIdenticalVersionedFile(source, files)
2395
def test_insert_record_stream_annotated_knits(self):
2396
"""Any file should accept a stream from plain knits."""
2397
files = self.get_versionedfiles()
2398
mapper = self.get_mapper()
2399
source_transport = self.get_transport('source')
2400
source_transport.mkdir('.')
2401
source = make_file_factory(True, mapper)(source_transport)
2402
self.get_diamond_files(source)
2403
stream = source.get_record_stream(source.keys(), 'topological',
2405
files.insert_record_stream(stream)
2406
self.assertIdenticalVersionedFile(source, files)
2408
def test_insert_record_stream_annotated_knits_noeol(self):
2409
"""Any file should accept a stream from plain knits."""
2410
files = self.get_versionedfiles()
2411
mapper = self.get_mapper()
2412
source_transport = self.get_transport('source')
2413
source_transport.mkdir('.')
2414
source = make_file_factory(True, mapper)(source_transport)
2415
self.get_diamond_files(source, trailing_eol=False)
2416
stream = source.get_record_stream(source.keys(), 'topological',
2418
files.insert_record_stream(stream)
2419
self.assertIdenticalVersionedFile(source, files)
2421
def test_insert_record_stream_plain_knits(self):
2422
"""Any file should accept a stream from plain knits."""
2423
files = self.get_versionedfiles()
2424
mapper = self.get_mapper()
2425
source_transport = self.get_transport('source')
2426
source_transport.mkdir('.')
2427
source = make_file_factory(False, mapper)(source_transport)
2428
self.get_diamond_files(source)
2429
stream = source.get_record_stream(source.keys(), 'topological',
2431
files.insert_record_stream(stream)
2432
self.assertIdenticalVersionedFile(source, files)
2434
def test_insert_record_stream_plain_knits_noeol(self):
2435
"""Any file should accept a stream from plain knits."""
2436
files = self.get_versionedfiles()
2437
mapper = self.get_mapper()
2438
source_transport = self.get_transport('source')
2439
source_transport.mkdir('.')
2440
source = make_file_factory(False, mapper)(source_transport)
2441
self.get_diamond_files(source, trailing_eol=False)
2442
stream = source.get_record_stream(source.keys(), 'topological',
2444
files.insert_record_stream(stream)
2445
self.assertIdenticalVersionedFile(source, files)
2447
def test_insert_record_stream_existing_keys(self):
2448
"""Inserting keys already in a file should not error."""
2449
files = self.get_versionedfiles()
2450
source = self.get_versionedfiles('source')
2451
self.get_diamond_files(source)
2452
# insert some keys into f.
2453
self.get_diamond_files(files, left_only=True)
2454
stream = source.get_record_stream(source.keys(), 'topological',
2456
files.insert_record_stream(stream)
2457
self.assertIdenticalVersionedFile(source, files)
2459
def test_insert_record_stream_missing_keys(self):
2460
"""Inserting a stream with absent keys should raise an error."""
2461
files = self.get_versionedfiles()
2462
source = self.get_versionedfiles('source')
2463
stream = source.get_record_stream([(b'missing',) * self.key_length],
2464
'topological', False)
2465
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2468
def test_insert_record_stream_out_of_order(self):
2469
"""An out of order stream can either error or work."""
2470
files = self.get_versionedfiles()
2471
source = self.get_versionedfiles('source')
2472
self.get_diamond_files(source)
2473
if self.key_length == 1:
2474
origin_keys = [(b'origin',)]
2475
end_keys = [(b'merged',), (b'left',)]
2476
start_keys = [(b'right',), (b'base',)]
2478
origin_keys = [(b'FileA', b'origin'), (b'FileB', b'origin')]
2479
end_keys = [(b'FileA', b'merged',), (b'FileA', b'left',),
2480
(b'FileB', b'merged',), (b'FileB', b'left',)]
2481
start_keys = [(b'FileA', b'right',), (b'FileA', b'base',),
2482
(b'FileB', b'right',), (b'FileB', b'base',)]
2483
origin_entries = source.get_record_stream(
2484
origin_keys, 'unordered', False)
2485
end_entries = source.get_record_stream(end_keys, 'topological', False)
2486
start_entries = source.get_record_stream(
2487
start_keys, 'topological', False)
2488
entries = itertools.chain(origin_entries, end_entries, start_entries)
2490
files.insert_record_stream(entries)
2491
except RevisionNotPresent:
2492
# Must not have corrupted the file.
2495
self.assertIdenticalVersionedFile(source, files)
2497
def test_insert_record_stream_long_parent_chain_out_of_order(self):
2498
"""An out of order stream can either error or work."""
2500
raise TestNotApplicable('ancestry info only relevant with graph.')
2501
# Create a reasonably long chain of records based on each other, where
2502
# most will be deltas.
2503
source = self.get_versionedfiles('source')
2506
content = [(b'same same %d\n' % n) for n in range(500)]
2507
letters = b'abcdefghijklmnopqrstuvwxyz'
2508
for i in range(len(letters)):
2509
letter = letters[i:i + 1]
2510
key = (b'key-' + letter,)
2511
if self.key_length == 2:
2512
key = (b'prefix',) + key
2513
content.append(b'content for ' + letter + b'\n')
2514
source.add_lines(key, parents, content)
2517
# Create a stream of these records, excluding the first record that the
2518
# rest ultimately depend upon, and insert it into a new vf.
2520
for key in reversed(keys):
2521
streams.append(source.get_record_stream([key], 'unordered', False))
2522
deltas = itertools.chain.from_iterable(streams[:-1])
2523
files = self.get_versionedfiles()
2525
files.insert_record_stream(deltas)
2526
except RevisionNotPresent:
2527
# Must not have corrupted the file.
2530
# Must only report either just the first key as a missing parent,
2531
# no key as missing (for nodelta scenarios).
2532
missing = set(files.get_missing_compression_parent_keys())
2533
missing.discard(keys[0])
2534
self.assertEqual(set(), missing)
2536
def get_knit_delta_source(self):
2537
"""Get a source that can produce a stream with knit delta records,
2538
regardless of this test's scenario.
2540
mapper = self.get_mapper()
2541
source_transport = self.get_transport('source')
2542
source_transport.mkdir('.')
2543
source = make_file_factory(False, mapper)(source_transport)
2544
get_diamond_files(source, self.key_length, trailing_eol=True,
2545
nograph=False, left_only=False)
2548
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2549
"""Insertion where a needed basis is not included notifies the caller
2550
of the missing basis. In the meantime a record missing its basis is
2553
source = self.get_knit_delta_source()
2554
keys = [self.get_simple_key(b'origin'), self.get_simple_key(b'merged')]
2555
entries = source.get_record_stream(keys, 'unordered', False)
2556
files = self.get_versionedfiles()
2557
if self.support_partial_insertion:
2558
self.assertEqual([],
2559
list(files.get_missing_compression_parent_keys()))
2560
files.insert_record_stream(entries)
2561
missing_bases = files.get_missing_compression_parent_keys()
2562
self.assertEqual({self.get_simple_key(b'left')},
2564
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2567
errors.RevisionNotPresent, files.insert_record_stream, entries)
2570
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2571
"""Insertion where a needed basis is not included notifies the caller
2572
of the missing basis. That basis can be added in a second
2573
insert_record_stream call that does not need to repeat records present
2574
in the previous stream. The record(s) that required that basis are
2575
fully inserted once their basis is no longer missing.
2577
if not self.support_partial_insertion:
2578
raise TestNotApplicable(
2579
'versioned file scenario does not support partial insertion')
2580
source = self.get_knit_delta_source()
2581
entries = source.get_record_stream([self.get_simple_key(b'origin'),
2582
self.get_simple_key(b'merged')], 'unordered', False)
2583
files = self.get_versionedfiles()
2584
files.insert_record_stream(entries)
2585
missing_bases = files.get_missing_compression_parent_keys()
2586
self.assertEqual({self.get_simple_key(b'left')},
2588
# 'merged' is inserted (although a commit of a write group involving
2589
# this versionedfiles would fail).
2590
merged_key = self.get_simple_key(b'merged')
2592
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2593
# Add the full delta closure of the missing records
2594
missing_entries = source.get_record_stream(
2595
missing_bases, 'unordered', True)
2596
files.insert_record_stream(missing_entries)
2597
# Now 'merged' is fully inserted (and a commit would succeed).
2598
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2600
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2603
def test_iter_lines_added_or_present_in_keys(self):
2604
# test that we get at least an equalset of the lines added by
2605
# versions in the store.
2606
# the ordering here is to make a tree so that dumb searches have
2607
# more changes to muck up.
2609
class InstrumentedProgress(progress.ProgressTask):
2612
progress.ProgressTask.__init__(self)
2615
def update(self, msg=None, current=None, total=None):
2616
self.updates.append((msg, current, total))
2618
files = self.get_versionedfiles()
2619
# add a base to get included
2620
files.add_lines(self.get_simple_key(b'base'), (), [b'base\n'])
2621
# add a ancestor to be included on one side
2622
files.add_lines(self.get_simple_key(
2623
b'lancestor'), (), [b'lancestor\n'])
2624
# add a ancestor to be included on the other side
2625
files.add_lines(self.get_simple_key(b'rancestor'),
2626
self.get_parents([self.get_simple_key(b'base')]), [b'rancestor\n'])
2627
# add a child of rancestor with no eofile-nl
2628
files.add_lines(self.get_simple_key(b'child'),
2629
self.get_parents([self.get_simple_key(b'rancestor')]),
2630
[b'base\n', b'child\n'])
2631
# add a child of lancestor and base to join the two roots
2632
files.add_lines(self.get_simple_key(b'otherchild'),
2633
self.get_parents([self.get_simple_key(b'lancestor'),
2634
self.get_simple_key(b'base')]),
2635
[b'base\n', b'lancestor\n', b'otherchild\n'])
2637
def iter_with_keys(keys, expected):
2638
# now we need to see what lines are returned, and how often.
2640
progress = InstrumentedProgress()
2641
# iterate over the lines
2642
for line in files.iter_lines_added_or_present_in_keys(keys,
2644
lines.setdefault(line, 0)
2646
if [] != progress.updates:
2647
self.assertEqual(expected, progress.updates)
2649
lines = iter_with_keys(
2650
[self.get_simple_key(b'child'),
2651
self.get_simple_key(b'otherchild')],
2652
[('Walking content', 0, 2),
2653
('Walking content', 1, 2),
2654
('Walking content', 2, 2)])
2655
# we must see child and otherchild
2656
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2658
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2659
# we dont care if we got more than that.
2662
lines = iter_with_keys(files.keys(),
2663
[('Walking content', 0, 5),
2664
('Walking content', 1, 5),
2665
('Walking content', 2, 5),
2666
('Walking content', 3, 5),
2667
('Walking content', 4, 5),
2668
('Walking content', 5, 5)])
2669
# all lines must be seen at least once
2670
self.assertTrue(lines[(b'base\n', self.get_simple_key(b'base'))] > 0)
2672
lines[(b'lancestor\n', self.get_simple_key(b'lancestor'))] > 0)
2674
lines[(b'rancestor\n', self.get_simple_key(b'rancestor'))] > 0)
2675
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2677
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2679
def test_make_mpdiffs(self):
2680
from breezy import multiparent
2681
files = self.get_versionedfiles('source')
2682
# add texts that should trip the knit maximum delta chain threshold
2683
# as well as doing parallel chains of data in knits.
2684
# this is done by two chains of 25 insertions
2685
files.add_lines(self.get_simple_key(b'base'), [], [b'line\n'])
2686
files.add_lines(self.get_simple_key(b'noeol'),
2687
self.get_parents([self.get_simple_key(b'base')]), [b'line'])
2688
# detailed eol tests:
2689
# shared last line with parent no-eol
2690
files.add_lines(self.get_simple_key(b'noeolsecond'),
2691
self.get_parents([self.get_simple_key(b'noeol')]),
2692
[b'line\n', b'line'])
2693
# differing last line with parent, both no-eol
2694
files.add_lines(self.get_simple_key(b'noeolnotshared'),
2696
[self.get_simple_key(b'noeolsecond')]),
2697
[b'line\n', b'phone'])
2698
# add eol following a noneol parent, change content
2699
files.add_lines(self.get_simple_key(b'eol'),
2700
self.get_parents([self.get_simple_key(b'noeol')]), [b'phone\n'])
2701
# add eol following a noneol parent, no change content
2702
files.add_lines(self.get_simple_key(b'eolline'),
2703
self.get_parents([self.get_simple_key(b'noeol')]), [b'line\n'])
2704
# noeol with no parents:
2705
files.add_lines(self.get_simple_key(b'noeolbase'), [], [b'line'])
2706
# noeol preceeding its leftmost parent in the output:
2707
# this is done by making it a merge of two parents with no common
2708
# anestry: noeolbase and noeol with the
2709
# later-inserted parent the leftmost.
2710
files.add_lines(self.get_simple_key(b'eolbeforefirstparent'),
2711
self.get_parents([self.get_simple_key(b'noeolbase'),
2712
self.get_simple_key(b'noeol')]),
2714
# two identical eol texts
2715
files.add_lines(self.get_simple_key(b'noeoldup'),
2716
self.get_parents([self.get_simple_key(b'noeol')]), [b'line'])
2717
next_parent = self.get_simple_key(b'base')
2718
text_name = b'chain1-'
2720
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2721
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2722
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2723
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2724
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
2725
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
2726
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2727
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2728
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
2729
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2730
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2731
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2732
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
2733
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2734
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
2735
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2736
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
2737
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2738
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
2739
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
2740
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2741
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2742
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2743
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2744
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2745
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2747
for depth in range(26):
2748
new_version = self.get_simple_key(text_name + b'%d' % depth)
2749
text = text + [b'line\n']
2750
files.add_lines(new_version, self.get_parents([next_parent]), text)
2751
next_parent = new_version
2752
next_parent = self.get_simple_key(b'base')
2753
text_name = b'chain2-'
2755
for depth in range(26):
2756
new_version = self.get_simple_key(text_name + b'%d' % depth)
2757
text = text + [b'line\n']
2758
files.add_lines(new_version, self.get_parents([next_parent]), text)
2759
next_parent = new_version
2760
target = self.get_versionedfiles('target')
2761
for key in multiparent.topo_iter_keys(files, files.keys()):
2762
mpdiff = files.make_mpdiffs([key])[0]
2763
parents = files.get_parent_map([key])[key] or []
2765
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2766
self.assertEqualDiff(
2767
next(files.get_record_stream([key], 'unordered',
2768
True)).get_bytes_as('fulltext'),
2769
next(target.get_record_stream([key], 'unordered',
2770
True)).get_bytes_as('fulltext')
2773
def test_keys(self):
2774
# While use is discouraged, versions() is still needed by aspects of
2776
files = self.get_versionedfiles()
2777
self.assertEqual(set(), set(files.keys()))
2778
if self.key_length == 1:
2781
key = (b'foo', b'bar',)
2782
files.add_lines(key, (), [])
2783
self.assertEqual({key}, set(files.keys()))
2786
class VirtualVersionedFilesTests(TestCase):
2787
"""Basic tests for the VirtualVersionedFiles implementations."""
2789
def _get_parent_map(self, keys):
2792
if k in self._parent_map:
2793
ret[k] = self._parent_map[k]
2797
super(VirtualVersionedFilesTests, self).setUp()
2799
self._parent_map = {}
2800
self.texts = VirtualVersionedFiles(self._get_parent_map,
2803
def test_add_lines(self):
2804
self.assertRaises(NotImplementedError,
2805
self.texts.add_lines, b"foo", [], [])
2807
def test_add_mpdiffs(self):
2808
self.assertRaises(NotImplementedError,
2809
self.texts.add_mpdiffs, [])
2811
def test_check_noerrors(self):
2814
def test_insert_record_stream(self):
2815
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2818
def test_get_sha1s_nonexistent(self):
2819
self.assertEqual({}, self.texts.get_sha1s([(b"NONEXISTENT",)]))
2821
def test_get_sha1s(self):
2822
self._lines[b"key"] = [b"dataline1", b"dataline2"]
2823
self.assertEqual({(b"key",): osutils.sha_strings(self._lines[b"key"])},
2824
self.texts.get_sha1s([(b"key",)]))
2826
def test_get_parent_map(self):
2827
self._parent_map = {b"G": (b"A", b"B")}
2828
self.assertEqual({(b"G",): ((b"A",), (b"B",))},
2829
self.texts.get_parent_map([(b"G",), (b"L",)]))
2831
def test_get_record_stream(self):
2832
self._lines[b"A"] = [b"FOO", b"BAR"]
2833
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2835
self.assertEqual("chunked", record.storage_kind)
2836
self.assertEqual(b"FOOBAR", record.get_bytes_as("fulltext"))
2837
self.assertEqual([b"FOO", b"BAR"], record.get_bytes_as("chunked"))
2839
def test_get_record_stream_absent(self):
2840
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2842
self.assertEqual("absent", record.storage_kind)
2844
def test_iter_lines_added_or_present_in_keys(self):
2845
self._lines[b"A"] = [b"FOO", b"BAR"]
2846
self._lines[b"B"] = [b"HEY"]
2847
self._lines[b"C"] = [b"Alberta"]
2848
it = self.texts.iter_lines_added_or_present_in_keys([(b"A",), (b"B",)])
2849
self.assertEqual(sorted([(b"FOO", b"A"), (b"BAR", b"A"), (b"HEY", b"B")]),
2853
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2855
def get_ordering_vf(self, key_priority):
2856
builder = self.make_branch_builder('test')
2857
builder.start_series()
2858
builder.build_snapshot(None, [
2859
('add', ('', b'TREE_ROOT', 'directory', None))],
2861
builder.build_snapshot([b'A'], [], revision_id=b'B')
2862
builder.build_snapshot([b'B'], [], revision_id=b'C')
2863
builder.build_snapshot([b'C'], [], revision_id=b'D')
2864
builder.finish_series()
2865
b = builder.get_branch()
2867
self.addCleanup(b.unlock)
2868
vf = b.repository.inventories
2869
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2871
def test_get_empty(self):
2872
vf = self.get_ordering_vf({})
2873
self.assertEqual([], vf.calls)
2875
def test_get_record_stream_topological(self):
2876
vf = self.get_ordering_vf(
2877
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2878
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2879
keys = [r.key for r in vf.get_record_stream(request_keys,
2880
'topological', False)]
2881
# We should have gotten the keys in topological order
2882
self.assertEqual([(b'A',), (b'B',), (b'C',), (b'D',)], keys)
2883
# And recorded that the request was made
2884
self.assertEqual([('get_record_stream', request_keys, 'topological',
2887
def test_get_record_stream_ordered(self):
2888
vf = self.get_ordering_vf(
2889
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2890
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2891
keys = [r.key for r in vf.get_record_stream(request_keys,
2892
'unordered', False)]
2893
# They should be returned based on their priority
2894
self.assertEqual([(b'D',), (b'B',), (b'A',), (b'C',)], keys)
2895
# And the request recorded
2896
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2899
def test_get_record_stream_implicit_order(self):
2900
vf = self.get_ordering_vf({(b'B',): 2, (b'D',): 1})
2901
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2902
keys = [r.key for r in vf.get_record_stream(request_keys,
2903
'unordered', False)]
2904
# A and C are not in the map, so they get sorted to the front. A comes
2905
# before C alphabetically, so it comes back first
2906
self.assertEqual([(b'A',), (b'C',), (b'D',), (b'B',)], keys)
2907
# And the request recorded
2908
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
breezy/tests/per_versionedfile.py
