1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
24
from gzip import GzipFile
39
from ..errors import (
41
RevisionAlreadyPresent,
43
from ..bzr.knit import (
48
from ..sixish import (
54
TestCaseWithMemoryTransport,
58
from .http_utils import TestCaseWithWebserver
59
from ..transport.memory import MemoryTransport
60
from ..bzr import versionedfile as versionedfile
61
from ..bzr.versionedfile import (
63
HashEscapedPrefixMapper,
65
VirtualVersionedFiles,
66
make_versioned_files_factory,
68
from ..bzr.weave import (
72
from ..bzr.weavefile import write_weave
73
from .scenarios import load_tests_apply_scenarios
76
load_tests = load_tests_apply_scenarios
79
def get_diamond_vf(f, trailing_eol=True, left_only=False):
80
"""Get a diamond graph to exercise deltas and merges.
82
:param trailing_eol: If True end the last line with \n.
86
b'base': ((b'origin',),),
87
b'left': ((b'base',),),
88
b'right': ((b'base',),),
89
b'merged': ((b'left',), (b'right',)),
91
# insert a diamond graph to exercise deltas and merges.
96
f.add_lines(b'origin', [], [b'origin' + last_char])
97
f.add_lines(b'base', [b'origin'], [b'base' + last_char])
98
f.add_lines(b'left', [b'base'], [b'base\n', b'left' + last_char])
100
f.add_lines(b'right', [b'base'],
101
[b'base\n', b'right' + last_char])
102
f.add_lines(b'merged', [b'left', b'right'],
103
[b'base\n', b'left\n', b'right\n', b'merged' + last_char])
107
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
108
nograph=False, nokeys=False):
109
"""Get a diamond graph to exercise deltas and merges.
111
This creates a 5-node graph in files. If files supports 2-length keys two
112
graphs are made to exercise the support for multiple ids.
114
:param trailing_eol: If True end the last line with \n.
115
:param key_length: The length of keys in files. Currently supports length 1
117
:param left_only: If True do not add the right and merged nodes.
118
:param nograph: If True, do not provide parents to the add_lines calls;
119
this is useful for tests that need inserted data but have graphless
121
:param nokeys: If True, pass None is as the key for all insertions.
122
Currently implies nograph.
123
:return: The results of the add_lines calls.
130
prefixes = [(b'FileA',), (b'FileB',)]
131
# insert a diamond graph to exercise deltas and merges.
138
def get_parents(suffix_list):
142
result = [prefix + suffix for suffix in suffix_list]
150
# we loop over each key because that spreads the inserts across prefixes,
151
# which is how commit operates.
152
for prefix in prefixes:
153
result.append(files.add_lines(prefix + get_key(b'origin'), (),
154
[b'origin' + last_char]))
155
for prefix in prefixes:
156
result.append(files.add_lines(prefix + get_key(b'base'),
157
get_parents([(b'origin',)]), [b'base' + last_char]))
158
for prefix in prefixes:
159
result.append(files.add_lines(prefix + get_key(b'left'),
160
get_parents([(b'base',)]),
161
[b'base\n', b'left' + last_char]))
163
for prefix in prefixes:
164
result.append(files.add_lines(prefix + get_key(b'right'),
165
get_parents([(b'base',)]),
166
[b'base\n', b'right' + last_char]))
167
for prefix in prefixes:
168
result.append(files.add_lines(prefix + get_key(b'merged'),
170
[(b'left',), (b'right',)]),
171
[b'base\n', b'left\n', b'right\n', b'merged' + last_char]))
175
class VersionedFileTestMixIn(object):
176
"""A mixin test class for testing VersionedFiles.
178
This is not an adaptor-style test at this point because
179
theres no dynamic substitution of versioned file implementations,
180
they are strictly controlled by their owning repositories.
183
def get_transaction(self):
184
if not hasattr(self, '_transaction'):
185
self._transaction = None
186
return self._transaction
190
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
191
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
194
versions = f.versions()
195
self.assertTrue(b'r0' in versions)
196
self.assertTrue(b'r1' in versions)
197
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
198
self.assertEqual(f.get_text(b'r0'), b'a\nb\n')
199
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
200
self.assertEqual(2, len(f))
201
self.assertEqual(2, f.num_versions())
203
self.assertRaises(RevisionNotPresent,
204
f.add_lines, b'r2', [b'foo'], [])
205
self.assertRaises(RevisionAlreadyPresent,
206
f.add_lines, b'r1', [], [])
208
# this checks that reopen with create=True does not break anything.
209
f = self.reopen_file(create=True)
212
def test_adds_with_parent_texts(self):
215
_, _, parent_texts[b'r0'] = f.add_lines(b'r0', [], [b'a\n', b'b\n'])
217
_, _, parent_texts[b'r1'] = f.add_lines_with_ghosts(b'r1',
218
[b'r0', b'ghost'], [b'b\n', b'c\n'], parent_texts=parent_texts)
219
except NotImplementedError:
220
# if the format doesn't support ghosts, just add normally.
221
_, _, parent_texts[b'r1'] = f.add_lines(b'r1',
222
[b'r0'], [b'b\n', b'c\n'], parent_texts=parent_texts)
223
f.add_lines(b'r2', [b'r1'], [b'c\n', b'd\n'],
224
parent_texts=parent_texts)
225
self.assertNotEqual(None, parent_texts[b'r0'])
226
self.assertNotEqual(None, parent_texts[b'r1'])
229
versions = f.versions()
230
self.assertTrue(b'r0' in versions)
231
self.assertTrue(b'r1' in versions)
232
self.assertTrue(b'r2' in versions)
233
self.assertEqual(f.get_lines(b'r0'), [b'a\n', b'b\n'])
234
self.assertEqual(f.get_lines(b'r1'), [b'b\n', b'c\n'])
235
self.assertEqual(f.get_lines(b'r2'), [b'c\n', b'd\n'])
236
self.assertEqual(3, f.num_versions())
237
origins = f.annotate(b'r1')
238
self.assertEqual(origins[0][0], b'r0')
239
self.assertEqual(origins[1][0], b'r1')
240
origins = f.annotate(b'r2')
241
self.assertEqual(origins[0][0], b'r1')
242
self.assertEqual(origins[1][0], b'r2')
245
f = self.reopen_file()
248
def test_add_unicode_content(self):
249
# unicode content is not permitted in versioned files.
250
# versioned files version sequences of bytes only.
252
self.assertRaises(errors.BzrBadParameterUnicode,
253
vf.add_lines, b'a', [], [b'a\n', u'b\n', b'c\n'])
255
(errors.BzrBadParameterUnicode, NotImplementedError),
256
vf.add_lines_with_ghosts, b'a', [], [b'a\n', u'b\n', b'c\n'])
258
def test_add_follows_left_matching_blocks(self):
259
"""If we change left_matching_blocks, delta changes
261
Note: There are multiple correct deltas in this case, because
262
we start with 1 "a" and we get 3.
265
if isinstance(vf, WeaveFile):
266
raise TestSkipped("WeaveFile ignores left_matching_blocks")
267
vf.add_lines(b'1', [], [b'a\n'])
268
vf.add_lines(b'2', [b'1'], [b'a\n', b'a\n', b'a\n'],
269
left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
270
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'2'))
271
vf.add_lines(b'3', [b'1'], [b'a\n', b'a\n', b'a\n'],
272
left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
273
self.assertEqual([b'a\n', b'a\n', b'a\n'], vf.get_lines(b'3'))
275
def test_inline_newline_throws(self):
276
# \r characters are not permitted in lines being added
278
self.assertRaises(errors.BzrBadParameterContainsNewline,
279
vf.add_lines, b'a', [], [b'a\n\n'])
281
(errors.BzrBadParameterContainsNewline, NotImplementedError),
282
vf.add_lines_with_ghosts, b'a', [], [b'a\n\n'])
283
# but inline CR's are allowed
284
vf.add_lines(b'a', [], [b'a\r\n'])
286
vf.add_lines_with_ghosts(b'b', [], [b'a\r\n'])
287
except NotImplementedError:
290
def test_add_reserved(self):
292
self.assertRaises(errors.ReservedId,
293
vf.add_lines, b'a:', [], [b'a\n', b'b\n', b'c\n'])
295
def test_add_lines_nostoresha(self):
296
"""When nostore_sha is supplied using old content raises."""
298
empty_text = (b'a', [])
299
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
300
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
302
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
303
sha, _, _ = vf.add_lines(version, [], lines)
305
# we now have a copy of all the lines in the vf.
306
for sha, (version, lines) in zip(
307
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
308
self.assertRaises(errors.ExistingContent,
309
vf.add_lines, version + b"2", [], lines,
311
# and no new version should have been added.
312
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
315
def test_add_lines_with_ghosts_nostoresha(self):
316
"""When nostore_sha is supplied using old content raises."""
318
empty_text = (b'a', [])
319
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
320
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
322
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
323
sha, _, _ = vf.add_lines(version, [], lines)
325
# we now have a copy of all the lines in the vf.
326
# is the test applicable to this vf implementation?
328
vf.add_lines_with_ghosts(b'd', [], [])
329
except NotImplementedError:
330
raise TestSkipped("add_lines_with_ghosts is optional")
331
for sha, (version, lines) in zip(
332
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
333
self.assertRaises(errors.ExistingContent,
334
vf.add_lines_with_ghosts, version + b"2", [], lines,
336
# and no new version should have been added.
337
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
340
def test_add_lines_return_value(self):
341
# add_lines should return the sha1 and the text size.
343
empty_text = (b'a', [])
344
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
345
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
346
# check results for the three cases:
347
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
348
# the first two elements are the same for all versioned files:
349
# - the digest and the size of the text. For some versioned files
350
# additional data is returned in additional tuple elements.
351
result = vf.add_lines(version, [], lines)
352
self.assertEqual(3, len(result))
353
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
355
# parents should not affect the result:
356
lines = sample_text_nl[1]
357
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
358
vf.add_lines(b'd', [b'b', b'c'], lines)[0:2])
360
def test_get_reserved(self):
362
self.assertRaises(errors.ReservedId, vf.get_texts, [b'b:'])
363
self.assertRaises(errors.ReservedId, vf.get_lines, b'b:')
364
self.assertRaises(errors.ReservedId, vf.get_text, b'b:')
366
def test_add_unchanged_last_line_noeol_snapshot(self):
367
"""Add a text with an unchanged last line with no eol should work."""
368
# Test adding this in a number of chain lengths; because the interface
369
# for VersionedFile does not allow forcing a specific chain length, we
370
# just use a small base to get the first snapshot, then a much longer
371
# first line for the next add (which will make the third add snapshot)
372
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
373
# as a capped delta length, but ideally we would have some way of
374
# tuning the test to the store (e.g. keep going until a snapshot
376
for length in range(20):
378
vf = self.get_file('case-%d' % length)
381
for step in range(length):
382
version = prefix % step
383
lines = ([b'prelude \n'] * step) + [b'line']
384
vf.add_lines(version, parents, lines)
385
version_lines[version] = lines
387
vf.add_lines(b'no-eol', parents, [b'line'])
388
vf.get_texts(version_lines.keys())
389
self.assertEqualDiff(b'line', vf.get_text(b'no-eol'))
391
def test_get_texts_eol_variation(self):
392
# similar to the failure in <http://bugs.launchpad.net/234748>
394
sample_text_nl = [b"line\n"]
395
sample_text_no_nl = [b"line"]
402
lines = sample_text_nl
404
lines = sample_text_no_nl
405
# left_matching blocks is an internal api; it operates on the
406
# *internal* representation for a knit, which is with *all* lines
407
# being normalised to end with \n - even the final line in a no_nl
408
# file. Using it here ensures that a broken internal implementation
409
# (which is what this test tests) will generate a correct line
410
# delta (which is to say, an empty delta).
411
vf.add_lines(version, parents, lines,
412
left_matching_blocks=[(0, 0, 1)])
414
versions.append(version)
415
version_lines[version] = lines
417
vf.get_texts(versions)
418
vf.get_texts(reversed(versions))
420
def test_add_lines_with_matching_blocks_noeol_last_line(self):
421
"""Add a text with an unchanged last line with no eol should work."""
422
from breezy import multiparent
423
# Hand verified sha1 of the text we're adding.
424
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
425
# Create a mpdiff which adds a new line before the trailing line, and
426
# reuse the last line unaltered (which can cause annotation reuse).
427
# Test adding this in two situations:
428
# On top of a new insertion
429
vf = self.get_file('fulltext')
430
vf.add_lines(b'noeol', [], [b'line'])
431
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
432
left_matching_blocks=[(0, 1, 1)])
433
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
435
vf = self.get_file('delta')
436
vf.add_lines(b'base', [], [b'line'])
437
vf.add_lines(b'noeol', [b'base'], [b'prelude\n', b'line'])
438
vf.add_lines(b'noeol2', [b'noeol'], [b'newline\n', b'line'],
439
left_matching_blocks=[(1, 1, 1)])
440
self.assertEqualDiff(b'newline\nline', vf.get_text(b'noeol2'))
442
def test_make_mpdiffs(self):
443
from breezy import multiparent
444
vf = self.get_file('foo')
445
sha1s = self._setup_for_deltas(vf)
446
new_vf = self.get_file('bar')
447
for version in multiparent.topo_iter(vf):
448
mpdiff = vf.make_mpdiffs([version])[0]
449
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
450
vf.get_sha1s([version])[version], mpdiff)])
451
self.assertEqualDiff(vf.get_text(version),
452
new_vf.get_text(version))
454
def test_make_mpdiffs_with_ghosts(self):
455
vf = self.get_file('foo')
457
vf.add_lines_with_ghosts(b'text', [b'ghost'], [b'line\n'])
458
except NotImplementedError:
459
# old Weave formats do not allow ghosts
461
self.assertRaises(errors.RevisionNotPresent,
462
vf.make_mpdiffs, [b'ghost'])
464
def _setup_for_deltas(self, f):
465
self.assertFalse(f.has_version('base'))
466
# add texts that should trip the knit maximum delta chain threshold
467
# as well as doing parallel chains of data in knits.
468
# this is done by two chains of 25 insertions
469
f.add_lines(b'base', [], [b'line\n'])
470
f.add_lines(b'noeol', [b'base'], [b'line'])
471
# detailed eol tests:
472
# shared last line with parent no-eol
473
f.add_lines(b'noeolsecond', [b'noeol'], [b'line\n', b'line'])
474
# differing last line with parent, both no-eol
475
f.add_lines(b'noeolnotshared', [b'noeolsecond'], [b'line\n', b'phone'])
476
# add eol following a noneol parent, change content
477
f.add_lines(b'eol', [b'noeol'], [b'phone\n'])
478
# add eol following a noneol parent, no change content
479
f.add_lines(b'eolline', [b'noeol'], [b'line\n'])
480
# noeol with no parents:
481
f.add_lines(b'noeolbase', [], [b'line'])
482
# noeol preceeding its leftmost parent in the output:
483
# this is done by making it a merge of two parents with no common
484
# anestry: noeolbase and noeol with the
485
# later-inserted parent the leftmost.
486
f.add_lines(b'eolbeforefirstparent', [
487
b'noeolbase', b'noeol'], [b'line'])
488
# two identical eol texts
489
f.add_lines(b'noeoldup', [b'noeol'], [b'line'])
490
next_parent = b'base'
491
text_name = b'chain1-'
493
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
494
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
495
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
496
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
497
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
498
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
499
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
500
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
501
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
502
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
503
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
504
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
505
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
506
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
507
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
508
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
509
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
510
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
511
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
512
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
513
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
514
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
515
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
516
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
517
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
518
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
520
for depth in range(26):
521
new_version = text_name + b'%d' % depth
522
text = text + [b'line\n']
523
f.add_lines(new_version, [next_parent], text)
524
next_parent = new_version
525
next_parent = b'base'
526
text_name = b'chain2-'
528
for depth in range(26):
529
new_version = text_name + b'%d' % depth
530
text = text + [b'line\n']
531
f.add_lines(new_version, [next_parent], text)
532
next_parent = new_version
535
def test_ancestry(self):
537
self.assertEqual([], f.get_ancestry([]))
538
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
539
f.add_lines(b'r1', [b'r0'], [b'b\n', b'c\n'])
540
f.add_lines(b'r2', [b'r0'], [b'b\n', b'c\n'])
541
f.add_lines(b'r3', [b'r2'], [b'b\n', b'c\n'])
542
f.add_lines(b'rM', [b'r1', b'r2'], [b'b\n', b'c\n'])
543
self.assertEqual([], f.get_ancestry([]))
544
versions = f.get_ancestry([b'rM'])
545
# there are some possibilities:
549
# so we check indexes
550
r0 = versions.index(b'r0')
551
r1 = versions.index(b'r1')
552
r2 = versions.index(b'r2')
553
self.assertFalse(b'r3' in versions)
554
rM = versions.index(b'rM')
555
self.assertTrue(r0 < r1)
556
self.assertTrue(r0 < r2)
557
self.assertTrue(r1 < rM)
558
self.assertTrue(r2 < rM)
560
self.assertRaises(RevisionNotPresent,
561
f.get_ancestry, [b'rM', b'rX'])
563
self.assertEqual(set(f.get_ancestry(b'rM')),
564
set(f.get_ancestry(b'rM', topo_sorted=False)))
566
def test_mutate_after_finish(self):
567
self._transaction = 'before'
569
self._transaction = 'after'
570
self.assertRaises(errors.OutSideTransaction, f.add_lines, b'', [], [])
571
self.assertRaises(errors.OutSideTransaction,
572
f.add_lines_with_ghosts, b'', [], [])
574
def test_copy_to(self):
576
f.add_lines(b'0', [], [b'a\n'])
577
t = MemoryTransport()
579
for suffix in self.get_factory().get_suffixes():
580
self.assertTrue(t.has('foo' + suffix))
582
def test_get_suffixes(self):
584
# and should be a list
585
self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
587
def test_get_parent_map(self):
589
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
591
{b'r0': ()}, f.get_parent_map([b'r0']))
592
f.add_lines(b'r1', [b'r0'], [b'a\n', b'b\n'])
594
{b'r1': (b'r0',)}, f.get_parent_map([b'r1']))
598
f.get_parent_map([b'r0', b'r1']))
599
f.add_lines(b'r2', [], [b'a\n', b'b\n'])
600
f.add_lines(b'r3', [], [b'a\n', b'b\n'])
601
f.add_lines(b'm', [b'r0', b'r1', b'r2', b'r3'], [b'a\n', b'b\n'])
603
{b'm': (b'r0', b'r1', b'r2', b'r3')}, f.get_parent_map([b'm']))
604
self.assertEqual({}, f.get_parent_map(b'y'))
608
f.get_parent_map([b'r0', b'y', b'r1']))
610
def test_annotate(self):
612
f.add_lines(b'r0', [], [b'a\n', b'b\n'])
613
f.add_lines(b'r1', [b'r0'], [b'c\n', b'b\n'])
614
origins = f.annotate(b'r1')
615
self.assertEqual(origins[0][0], b'r1')
616
self.assertEqual(origins[1][0], b'r0')
618
self.assertRaises(RevisionNotPresent,
621
def test_detection(self):
622
# Test weaves detect corruption.
624
# Weaves contain a checksum of their texts.
625
# When a text is extracted, this checksum should be
628
w = self.get_file_corrupted_text()
630
self.assertEqual(b'hello\n', w.get_text(b'v1'))
631
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
632
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
633
self.assertRaises(WeaveInvalidChecksum, w.check)
635
w = self.get_file_corrupted_checksum()
637
self.assertEqual(b'hello\n', w.get_text(b'v1'))
638
self.assertRaises(WeaveInvalidChecksum, w.get_text, b'v2')
639
self.assertRaises(WeaveInvalidChecksum, w.get_lines, b'v2')
640
self.assertRaises(WeaveInvalidChecksum, w.check)
642
def get_file_corrupted_text(self):
643
"""Return a versioned file with corrupt text but valid metadata."""
644
raise NotImplementedError(self.get_file_corrupted_text)
646
def reopen_file(self, name='foo'):
647
"""Open the versioned file from disk again."""
648
raise NotImplementedError(self.reopen_file)
650
def test_iter_lines_added_or_present_in_versions(self):
651
# test that we get at least an equalset of the lines added by
652
# versions in the weave
653
# the ordering here is to make a tree so that dumb searches have
654
# more changes to muck up.
656
class InstrumentedProgress(progress.ProgressTask):
659
progress.ProgressTask.__init__(self)
662
def update(self, msg=None, current=None, total=None):
663
self.updates.append((msg, current, total))
666
# add a base to get included
667
vf.add_lines(b'base', [], [b'base\n'])
668
# add a ancestor to be included on one side
669
vf.add_lines(b'lancestor', [], [b'lancestor\n'])
670
# add a ancestor to be included on the other side
671
vf.add_lines(b'rancestor', [b'base'], [b'rancestor\n'])
672
# add a child of rancestor with no eofile-nl
673
vf.add_lines(b'child', [b'rancestor'], [b'base\n', b'child\n'])
674
# add a child of lancestor and base to join the two roots
675
vf.add_lines(b'otherchild',
676
[b'lancestor', b'base'],
677
[b'base\n', b'lancestor\n', b'otherchild\n'])
679
def iter_with_versions(versions, expected):
680
# now we need to see what lines are returned, and how often.
682
progress = InstrumentedProgress()
683
# iterate over the lines
684
for line in vf.iter_lines_added_or_present_in_versions(versions,
686
lines.setdefault(line, 0)
688
if [] != progress.updates:
689
self.assertEqual(expected, progress.updates)
691
lines = iter_with_versions([b'child', b'otherchild'],
692
[('Walking content', 0, 2),
693
('Walking content', 1, 2),
694
('Walking content', 2, 2)])
695
# we must see child and otherchild
696
self.assertTrue(lines[(b'child\n', b'child')] > 0)
697
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
698
# we dont care if we got more than that.
701
lines = iter_with_versions(None, [('Walking content', 0, 5),
702
('Walking content', 1, 5),
703
('Walking content', 2, 5),
704
('Walking content', 3, 5),
705
('Walking content', 4, 5),
706
('Walking content', 5, 5)])
707
# all lines must be seen at least once
708
self.assertTrue(lines[(b'base\n', b'base')] > 0)
709
self.assertTrue(lines[(b'lancestor\n', b'lancestor')] > 0)
710
self.assertTrue(lines[(b'rancestor\n', b'rancestor')] > 0)
711
self.assertTrue(lines[(b'child\n', b'child')] > 0)
712
self.assertTrue(lines[(b'otherchild\n', b'otherchild')] > 0)
714
def test_add_lines_with_ghosts(self):
715
# some versioned file formats allow lines to be added with parent
716
# information that is > than that in the format. Formats that do
717
# not support this need to raise NotImplementedError on the
718
# add_lines_with_ghosts api.
720
# add a revision with ghost parents
721
# The preferred form is utf8, but we should translate when needed
722
parent_id_unicode = u'b\xbfse'
723
parent_id_utf8 = parent_id_unicode.encode('utf8')
725
vf.add_lines_with_ghosts(b'notbxbfse', [parent_id_utf8], [])
726
except NotImplementedError:
727
# check the other ghost apis are also not implemented
728
self.assertRaises(NotImplementedError,
729
vf.get_ancestry_with_ghosts, [b'foo'])
730
self.assertRaises(NotImplementedError,
731
vf.get_parents_with_ghosts, b'foo')
733
vf = self.reopen_file()
734
# test key graph related apis: getncestry, _graph, get_parents
736
# - these are ghost unaware and must not be reflect ghosts
737
self.assertEqual([b'notbxbfse'], vf.get_ancestry(b'notbxbfse'))
738
self.assertFalse(vf.has_version(parent_id_utf8))
739
# we have _with_ghost apis to give us ghost information.
740
self.assertEqual([parent_id_utf8, b'notbxbfse'],
741
vf.get_ancestry_with_ghosts([b'notbxbfse']))
742
self.assertEqual([parent_id_utf8],
743
vf.get_parents_with_ghosts(b'notbxbfse'))
744
# if we add something that is a ghost of another, it should correct the
745
# results of the prior apis
746
vf.add_lines(parent_id_utf8, [], [])
747
self.assertEqual([parent_id_utf8, b'notbxbfse'],
748
vf.get_ancestry([b'notbxbfse']))
749
self.assertEqual({b'notbxbfse': (parent_id_utf8,)},
750
vf.get_parent_map([b'notbxbfse']))
751
self.assertTrue(vf.has_version(parent_id_utf8))
752
# we have _with_ghost apis to give us ghost information.
753
self.assertEqual([parent_id_utf8, b'notbxbfse'],
754
vf.get_ancestry_with_ghosts([b'notbxbfse']))
755
self.assertEqual([parent_id_utf8],
756
vf.get_parents_with_ghosts(b'notbxbfse'))
758
def test_add_lines_with_ghosts_after_normal_revs(self):
759
# some versioned file formats allow lines to be added with parent
760
# information that is > than that in the format. Formats that do
761
# not support this need to raise NotImplementedError on the
762
# add_lines_with_ghosts api.
764
# probe for ghost support
766
vf.add_lines_with_ghosts(b'base', [], [b'line\n', b'line_b\n'])
767
except NotImplementedError:
769
vf.add_lines_with_ghosts(b'references_ghost',
770
[b'base', b'a_ghost'],
771
[b'line\n', b'line_b\n', b'line_c\n'])
772
origins = vf.annotate(b'references_ghost')
773
self.assertEqual((b'base', b'line\n'), origins[0])
774
self.assertEqual((b'base', b'line_b\n'), origins[1])
775
self.assertEqual((b'references_ghost', b'line_c\n'), origins[2])
777
def test_readonly_mode(self):
778
t = self.get_transport()
779
factory = self.get_factory()
780
vf = factory('id', t, 0o777, create=True, access_mode='w')
781
vf = factory('id', t, access_mode='r')
782
self.assertRaises(errors.ReadOnlyError, vf.add_lines, b'base', [], [])
783
self.assertRaises(errors.ReadOnlyError,
784
vf.add_lines_with_ghosts,
789
def test_get_sha1s(self):
790
# check the sha1 data is available
793
vf.add_lines(b'a', [], [b'a\n'])
794
# the same file, different metadata
795
vf.add_lines(b'b', [b'a'], [b'a\n'])
796
# a file differing only in last newline.
797
vf.add_lines(b'c', [], [b'a'])
799
b'a': b'3f786850e387550fdab836ed7e6dc881de23001b',
800
b'c': b'86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
801
b'b': b'3f786850e387550fdab836ed7e6dc881de23001b',
803
vf.get_sha1s([b'a', b'c', b'b']))
806
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
808
def get_file(self, name='foo'):
809
return WeaveFile(name, self.get_transport(),
811
get_scope=self.get_transaction)
813
def get_file_corrupted_text(self):
814
w = WeaveFile('foo', self.get_transport(),
816
get_scope=self.get_transaction)
817
w.add_lines(b'v1', [], [b'hello\n'])
818
w.add_lines(b'v2', [b'v1'], [b'hello\n', b'there\n'])
820
# We are going to invasively corrupt the text
821
# Make sure the internals of weave are the same
822
self.assertEqual([(b'{', 0), b'hello\n', (b'}', None), (b'{', 1), b'there\n', (b'}', None)
825
self.assertEqual([b'f572d396fae9206628714fb2ce00f72e94f2258f', b'90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
830
w._weave[4] = b'There\n'
833
def get_file_corrupted_checksum(self):
834
w = self.get_file_corrupted_text()
836
w._weave[4] = b'there\n'
837
self.assertEqual(b'hello\nthere\n', w.get_text(b'v2'))
839
# Invalid checksum, first digit changed
840
w._sha1s[1] = b'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
843
def reopen_file(self, name='foo', create=False):
844
return WeaveFile(name, self.get_transport(),
846
get_scope=self.get_transaction)
848
def test_no_implicit_create(self):
849
self.assertRaises(errors.NoSuchFile,
852
self.get_transport(),
853
get_scope=self.get_transaction)
855
def get_factory(self):
859
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
862
super(TestPlanMergeVersionedFile, self).setUp()
863
mapper = PrefixMapper()
864
factory = make_file_factory(True, mapper)
865
self.vf1 = factory(self.get_transport('root-1'))
866
self.vf2 = factory(self.get_transport('root-2'))
867
self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
868
self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
870
def test_add_lines(self):
871
self.plan_merge_vf.add_lines((b'root', b'a:'), [], [])
872
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
873
(b'root', b'a'), [], [])
874
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
875
(b'root', b'a:'), None, [])
876
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
877
(b'root', b'a:'), [], None)
879
def setup_abcde(self):
880
self.vf1.add_lines((b'root', b'A'), [], [b'a'])
881
self.vf1.add_lines((b'root', b'B'), [(b'root', b'A')], [b'b'])
882
self.vf2.add_lines((b'root', b'C'), [], [b'c'])
883
self.vf2.add_lines((b'root', b'D'), [(b'root', b'C')], [b'd'])
884
self.plan_merge_vf.add_lines((b'root', b'E:'),
885
[(b'root', b'B'), (b'root', b'D')], [b'e'])
887
def test_get_parents(self):
889
self.assertEqual({(b'root', b'B'): ((b'root', b'A'),)},
890
self.plan_merge_vf.get_parent_map([(b'root', b'B')]))
891
self.assertEqual({(b'root', b'D'): ((b'root', b'C'),)},
892
self.plan_merge_vf.get_parent_map([(b'root', b'D')]))
893
self.assertEqual({(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D'))},
894
self.plan_merge_vf.get_parent_map([(b'root', b'E:')]))
896
self.plan_merge_vf.get_parent_map([(b'root', b'F')]))
898
(b'root', b'B'): ((b'root', b'A'),),
899
(b'root', b'D'): ((b'root', b'C'),),
900
(b'root', b'E:'): ((b'root', b'B'), (b'root', b'D')),
902
self.plan_merge_vf.get_parent_map(
903
[(b'root', b'B'), (b'root', b'D'), (b'root', b'E:'), (b'root', b'F')]))
905
def test_get_record_stream(self):
908
def get_record(suffix):
909
return next(self.plan_merge_vf.get_record_stream(
910
[(b'root', suffix)], 'unordered', True))
911
self.assertEqual(b'a', get_record(b'A').get_bytes_as('fulltext'))
912
self.assertEqual(b'c', get_record(b'C').get_bytes_as('fulltext'))
913
self.assertEqual(b'e', get_record(b'E:').get_bytes_as('fulltext'))
914
self.assertEqual('absent', get_record('F').storage_kind)
917
class TestReadonlyHttpMixin(object):
919
def get_transaction(self):
922
def test_readonly_http_works(self):
923
# we should be able to read from http with a versioned file.
925
# try an empty file access
926
readonly_vf = self.get_factory()('foo',
927
transport.get_transport_from_url(self.get_readonly_url('.')))
928
self.assertEqual([], readonly_vf.versions())
930
def test_readonly_http_works_with_feeling(self):
931
# we should be able to read from http with a versioned file.
934
vf.add_lines(b'1', [], [b'a\n'])
935
vf.add_lines(b'2', [b'1'], [b'b\n', b'a\n'])
936
readonly_vf = self.get_factory()('foo',
937
transport.get_transport_from_url(self.get_readonly_url('.')))
938
self.assertEqual([b'1', b'2'], vf.versions())
939
self.assertEqual([b'1', b'2'], readonly_vf.versions())
940
for version in readonly_vf.versions():
941
readonly_vf.get_lines(version)
944
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
947
return WeaveFile('foo', self.get_transport(),
949
get_scope=self.get_transaction)
951
def get_factory(self):
955
class MergeCasesMixin(object):
957
def doMerge(self, base, a, b, mp):
958
from textwrap import dedent
964
w.add_lines(b'text0', [], list(map(addcrlf, base)))
965
w.add_lines(b'text1', [b'text0'], list(map(addcrlf, a)))
966
w.add_lines(b'text2', [b'text0'], list(map(addcrlf, b)))
970
self.log('merge plan:')
971
p = list(w.plan_merge(b'text1', b'text2'))
972
for state, line in p:
974
self.log('%12s | %s' % (state, line[:-1]))
978
mt.writelines(w.weave_merge(p))
980
self.log(mt.getvalue())
982
mp = list(map(addcrlf, mp))
983
self.assertEqual(mt.readlines(), mp)
985
def testOneInsert(self):
991
def testSeparateInserts(self):
992
self.doMerge([b'aaa', b'bbb', b'ccc'],
993
[b'aaa', b'xxx', b'bbb', b'ccc'],
994
[b'aaa', b'bbb', b'yyy', b'ccc'],
995
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
997
def testSameInsert(self):
998
self.doMerge([b'aaa', b'bbb', b'ccc'],
999
[b'aaa', b'xxx', b'bbb', b'ccc'],
1000
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'],
1001
[b'aaa', b'xxx', b'bbb', b'yyy', b'ccc'])
1002
overlappedInsertExpected = [b'aaa', b'xxx', b'yyy', b'bbb']
1004
def testOverlappedInsert(self):
1005
self.doMerge([b'aaa', b'bbb'],
1006
[b'aaa', b'xxx', b'yyy', b'bbb'],
1007
[b'aaa', b'xxx', b'bbb'], self.overlappedInsertExpected)
1009
# really it ought to reduce this to
1010
# [b'aaa', b'xxx', b'yyy', b'bbb']
1012
def testClashReplace(self):
1013
self.doMerge([b'aaa'],
1016
[b'<<<<<<< ', b'xxx', b'=======', b'yyy', b'zzz',
1019
def testNonClashInsert1(self):
1020
self.doMerge([b'aaa'],
1023
[b'<<<<<<< ', b'xxx', b'aaa', b'=======', b'yyy', b'zzz',
1026
def testNonClashInsert2(self):
1027
self.doMerge([b'aaa'],
1032
def testDeleteAndModify(self):
1033
"""Clashing delete and modification.
1035
If one side modifies a region and the other deletes it then
1036
there should be a conflict with one side blank.
1039
#######################################
1040
# skippd, not working yet
1043
self.doMerge([b'aaa', b'bbb', b'ccc'],
1044
[b'aaa', b'ddd', b'ccc'],
1046
[b'<<<<<<<< ', b'aaa', b'=======', b'>>>>>>> ', b'ccc'])
1048
def _test_merge_from_strings(self, base, a, b, expected):
1050
w.add_lines(b'text0', [], base.splitlines(True))
1051
w.add_lines(b'text1', [b'text0'], a.splitlines(True))
1052
w.add_lines(b'text2', [b'text0'], b.splitlines(True))
1053
self.log('merge plan:')
1054
p = list(w.plan_merge(b'text1', b'text2'))
1055
for state, line in p:
1057
self.log('%12s | %s' % (state, line[:-1]))
1058
self.log('merge result:')
1059
result_text = b''.join(w.weave_merge(p))
1060
self.log(result_text)
1061
self.assertEqualDiff(result_text, expected)
1063
def test_weave_merge_conflicts(self):
1064
# does weave merge properly handle plans that end with unchanged?
1065
result = b''.join(self.get_file().weave_merge([('new-a', b'hello\n')]))
1066
self.assertEqual(result, b'hello\n')
1068
def test_deletion_extended(self):
1069
"""One side deletes, the other deletes more.
1090
self._test_merge_from_strings(base, a, b, result)
1092
def test_deletion_overlap(self):
1093
"""Delete overlapping regions with no other conflict.
1095
Arguably it'd be better to treat these as agreement, rather than
1096
conflict, but for now conflict is safer.
1124
self._test_merge_from_strings(base, a, b, result)
1126
def test_agreement_deletion(self):
1127
"""Agree to delete some lines, without conflicts."""
1149
self._test_merge_from_strings(base, a, b, result)
1151
def test_sync_on_deletion(self):
1152
"""Specific case of merge where we can synchronize incorrectly.
1154
A previous version of the weave merge concluded that the two versions
1155
agreed on deleting line 2, and this could be a synchronization point.
1156
Line 1 was then considered in isolation, and thought to be deleted on
1159
It's better to consider the whole thing as a disagreement region.
1170
a's replacement line 2
1183
a's replacement line 2
1190
self._test_merge_from_strings(base, a, b, result)
1193
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1195
def get_file(self, name='foo'):
1196
return WeaveFile(name, self.get_transport(),
1199
def log_contents(self, w):
1200
self.log('weave is:')
1202
write_weave(w, tmpf)
1203
self.log(tmpf.getvalue())
1205
overlappedInsertExpected = [b'aaa', b'<<<<<<< ', b'xxx', b'yyy', b'=======',
1206
b'xxx', b'>>>>>>> ', b'bbb']
1209
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1211
def test_select_adaptor(self):
1212
"""Test expected adapters exist."""
1213
# One scenario for each lookup combination we expect to use.
1214
# Each is source_kind, requested_kind, adapter class
1216
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1217
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1218
('knit-annotated-delta-gz', 'knit-delta-gz',
1219
_mod_knit.DeltaAnnotatedToUnannotated),
1220
('knit-annotated-delta-gz', 'fulltext',
1221
_mod_knit.DeltaAnnotatedToFullText),
1222
('knit-annotated-ft-gz', 'knit-ft-gz',
1223
_mod_knit.FTAnnotatedToUnannotated),
1224
('knit-annotated-ft-gz', 'fulltext',
1225
_mod_knit.FTAnnotatedToFullText),
1227
for source, requested, klass in scenarios:
1228
adapter_factory = versionedfile.adapter_registry.get(
1229
(source, requested))
1230
adapter = adapter_factory(None)
1231
self.assertIsInstance(adapter, klass)
1233
def get_knit(self, annotated=True):
1234
mapper = ConstantMapper('knit')
1235
transport = self.get_transport()
1236
return make_file_factory(annotated, mapper)(transport)
1238
def helpGetBytes(self, f, ft_adapter, delta_adapter):
1239
"""Grab the interested adapted texts for tests."""
1240
# origin is a fulltext
1241
entries = f.get_record_stream([(b'origin',)], 'unordered', False)
1242
base = next(entries)
1243
ft_data = ft_adapter.get_bytes(base)
1244
# merged is both a delta and multiple parents.
1245
entries = f.get_record_stream([(b'merged',)], 'unordered', False)
1246
merged = next(entries)
1247
delta_data = delta_adapter.get_bytes(merged)
1248
return ft_data, delta_data
1250
def test_deannotation_noeol(self):
1251
"""Test converting annotated knits to unannotated knits."""
1252
# we need a full text, and a delta
1254
get_diamond_files(f, 1, trailing_eol=False)
1255
ft_data, delta_data = self.helpGetBytes(f,
1256
_mod_knit.FTAnnotatedToUnannotated(
1258
_mod_knit.DeltaAnnotatedToUnannotated(None))
1260
b'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1263
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1265
b'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1266
b'1,2,3\nleft\nright\nmerged\nend merged\n',
1267
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1269
def test_deannotation(self):
1270
"""Test converting annotated knits to unannotated knits."""
1271
# we need a full text, and a delta
1273
get_diamond_files(f, 1)
1274
ft_data, delta_data = self.helpGetBytes(f,
1275
_mod_knit.FTAnnotatedToUnannotated(
1277
_mod_knit.DeltaAnnotatedToUnannotated(None))
1279
b'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1282
GzipFile(mode='rb', fileobj=BytesIO(ft_data)).read())
1284
b'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1285
b'2,2,2\nright\nmerged\nend merged\n',
1286
GzipFile(mode='rb', fileobj=BytesIO(delta_data)).read())
1288
def test_annotated_to_fulltext_no_eol(self):
1289
"""Test adapting annotated knits to full texts (for -> weaves)."""
1290
# we need a full text, and a delta
1292
get_diamond_files(f, 1, trailing_eol=False)
1293
# Reconstructing a full text requires a backing versioned file, and it
1294
# must have the base lines requested from it.
1295
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1296
ft_data, delta_data = self.helpGetBytes(f,
1297
_mod_knit.FTAnnotatedToFullText(
1299
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1300
self.assertEqual(b'origin', ft_data)
1301
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1302
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1303
True)], logged_vf.calls)
1305
def test_annotated_to_fulltext(self):
1306
"""Test adapting annotated knits to full texts (for -> weaves)."""
1307
# we need a full text, and a delta
1309
get_diamond_files(f, 1)
1310
# Reconstructing a full text requires a backing versioned file, and it
1311
# must have the base lines requested from it.
1312
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1313
ft_data, delta_data = self.helpGetBytes(f,
1314
_mod_knit.FTAnnotatedToFullText(
1316
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1317
self.assertEqual(b'origin\n', ft_data)
1318
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1319
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1320
True)], logged_vf.calls)
1322
def test_unannotated_to_fulltext(self):
1323
"""Test adapting unannotated knits to full texts.
1325
This is used for -> weaves, and for -> annotated knits.
1327
# we need a full text, and a delta
1328
f = self.get_knit(annotated=False)
1329
get_diamond_files(f, 1)
1330
# Reconstructing a full text requires a backing versioned file, and it
1331
# must have the base lines requested from it.
1332
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1333
ft_data, delta_data = self.helpGetBytes(f,
1334
_mod_knit.FTPlainToFullText(
1336
_mod_knit.DeltaPlainToFullText(logged_vf))
1337
self.assertEqual(b'origin\n', ft_data)
1338
self.assertEqual(b'base\nleft\nright\nmerged\n', delta_data)
1339
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1340
True)], logged_vf.calls)
1342
def test_unannotated_to_fulltext_no_eol(self):
1343
"""Test adapting unannotated knits to full texts.
1345
This is used for -> weaves, and for -> annotated knits.
1347
# we need a full text, and a delta
1348
f = self.get_knit(annotated=False)
1349
get_diamond_files(f, 1, trailing_eol=False)
1350
# Reconstructing a full text requires a backing versioned file, and it
1351
# must have the base lines requested from it.
1352
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1353
ft_data, delta_data = self.helpGetBytes(f,
1354
_mod_knit.FTPlainToFullText(
1356
_mod_knit.DeltaPlainToFullText(logged_vf))
1357
self.assertEqual(b'origin', ft_data)
1358
self.assertEqual(b'base\nleft\nright\nmerged', delta_data)
1359
self.assertEqual([('get_record_stream', [(b'left',)], 'unordered',
1360
True)], logged_vf.calls)
1363
class TestKeyMapper(TestCaseWithMemoryTransport):
1364
"""Tests for various key mapping logic."""
1366
def test_identity_mapper(self):
1367
mapper = versionedfile.ConstantMapper("inventory")
1368
self.assertEqual("inventory", mapper.map((b'foo@ar',)))
1369
self.assertEqual("inventory", mapper.map((b'quux',)))
1371
def test_prefix_mapper(self):
1373
mapper = versionedfile.PrefixMapper()
1374
self.assertEqual("file-id", mapper.map((b"file-id", b"revision-id")))
1375
self.assertEqual("new-id", mapper.map((b"new-id", b"revision-id")))
1376
self.assertEqual((b'file-id',), mapper.unmap("file-id"))
1377
self.assertEqual((b'new-id',), mapper.unmap("new-id"))
1379
def test_hash_prefix_mapper(self):
1380
#format6: hash + plain
1381
mapper = versionedfile.HashPrefixMapper()
1383
"9b/file-id", mapper.map((b"file-id", b"revision-id")))
1384
self.assertEqual("45/new-id", mapper.map((b"new-id", b"revision-id")))
1385
self.assertEqual((b'file-id',), mapper.unmap("9b/file-id"))
1386
self.assertEqual((b'new-id',), mapper.unmap("45/new-id"))
1388
def test_hash_escaped_mapper(self):
1389
#knit1: hash + escaped
1390
mapper = versionedfile.HashEscapedPrefixMapper()
1391
self.assertEqual("88/%2520", mapper.map((b" ", b"revision-id")))
1392
self.assertEqual("ed/fil%2545-%2549d", mapper.map((b"filE-Id",
1394
self.assertEqual("88/ne%2557-%2549d", mapper.map((b"neW-Id",
1396
self.assertEqual((b'filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1397
self.assertEqual((b'neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1400
class TestVersionedFiles(TestCaseWithMemoryTransport):
1401
"""Tests for the multiple-file variant of VersionedFile."""
1403
# We want to be sure of behaviour for:
1404
# weaves prefix layout (weave texts)
1405
# individually named weaves (weave inventories)
1406
# annotated knits - prefix|hash|hash-escape layout, we test the third only
1407
# as it is the most complex mapper.
1408
# individually named knits
1409
# individual no-graph knits in packs (signatures)
1410
# individual graph knits in packs (inventories)
1411
# individual graph nocompression knits in packs (revisions)
1412
# plain text knits in packs (texts)
1413
len_one_scenarios = [
1416
'factory': make_versioned_files_factory(WeaveFile,
1417
ConstantMapper('inventory')),
1420
'support_partial_insertion': False,
1424
'factory': make_file_factory(False, ConstantMapper('revisions')),
1427
'support_partial_insertion': False,
1429
('named-nograph-nodelta-knit-pack', {
1430
'cleanup': cleanup_pack_knit,
1431
'factory': make_pack_factory(False, False, 1),
1434
'support_partial_insertion': False,
1436
('named-graph-knit-pack', {
1437
'cleanup': cleanup_pack_knit,
1438
'factory': make_pack_factory(True, True, 1),
1441
'support_partial_insertion': True,
1443
('named-graph-nodelta-knit-pack', {
1444
'cleanup': cleanup_pack_knit,
1445
'factory': make_pack_factory(True, False, 1),
1448
'support_partial_insertion': False,
1450
('groupcompress-nograph', {
1451
'cleanup': groupcompress.cleanup_pack_group,
1452
'factory': groupcompress.make_pack_factory(False, False, 1),
1455
'support_partial_insertion': False,
1458
len_two_scenarios = [
1461
'factory': make_versioned_files_factory(WeaveFile,
1465
'support_partial_insertion': False,
1467
('annotated-knit-escape', {
1469
'factory': make_file_factory(True, HashEscapedPrefixMapper()),
1472
'support_partial_insertion': False,
1474
('plain-knit-pack', {
1475
'cleanup': cleanup_pack_knit,
1476
'factory': make_pack_factory(True, True, 2),
1479
'support_partial_insertion': True,
1482
'cleanup': groupcompress.cleanup_pack_group,
1483
'factory': groupcompress.make_pack_factory(True, False, 1),
1486
'support_partial_insertion': False,
1490
scenarios = len_one_scenarios + len_two_scenarios
1492
def get_versionedfiles(self, relpath='files'):
1493
transport = self.get_transport(relpath)
1495
transport.mkdir('.')
1496
files = self.factory(transport)
1497
if self.cleanup is not None:
1498
self.addCleanup(self.cleanup, files)
1501
def get_simple_key(self, suffix):
1502
"""Return a key for the object under test."""
1503
if self.key_length == 1:
1506
return (b'FileA',) + (suffix,)
1508
def test_add_fallback_implies_without_fallbacks(self):
1509
f = self.get_versionedfiles('files')
1510
if getattr(f, 'add_fallback_versioned_files', None) is None:
1511
raise TestNotApplicable("%s doesn't support fallbacks"
1512
% (f.__class__.__name__,))
1513
g = self.get_versionedfiles('fallback')
1514
key_a = self.get_simple_key(b'a')
1515
g.add_lines(key_a, [], [b'\n'])
1516
f.add_fallback_versioned_files(g)
1517
self.assertTrue(key_a in f.get_parent_map([key_a]))
1519
key_a in f.without_fallbacks().get_parent_map([key_a]))
1521
def test_add_lines(self):
1522
f = self.get_versionedfiles()
1523
key0 = self.get_simple_key(b'r0')
1524
key1 = self.get_simple_key(b'r1')
1525
key2 = self.get_simple_key(b'r2')
1526
keyf = self.get_simple_key(b'foo')
1527
f.add_lines(key0, [], [b'a\n', b'b\n'])
1529
f.add_lines(key1, [key0], [b'b\n', b'c\n'])
1531
f.add_lines(key1, [], [b'b\n', b'c\n'])
1533
self.assertTrue(key0 in keys)
1534
self.assertTrue(key1 in keys)
1536
for record in f.get_record_stream([key0, key1], 'unordered', True):
1537
records.append((record.key, record.get_bytes_as('fulltext')))
1539
self.assertEqual([(key0, b'a\nb\n'), (key1, b'b\nc\n')], records)
1541
def test_annotate(self):
1542
files = self.get_versionedfiles()
1543
self.get_diamond_files(files)
1544
if self.key_length == 1:
1547
prefix = (b'FileA',)
1548
# introduced full text
1549
origins = files.annotate(prefix + (b'origin',))
1551
(prefix + (b'origin',), b'origin\n')],
1554
origins = files.annotate(prefix + (b'base',))
1556
(prefix + (b'base',), b'base\n')],
1559
origins = files.annotate(prefix + (b'merged',))
1562
(prefix + (b'base',), b'base\n'),
1563
(prefix + (b'left',), b'left\n'),
1564
(prefix + (b'right',), b'right\n'),
1565
(prefix + (b'merged',), b'merged\n')
1569
# Without a graph everything is new.
1571
(prefix + (b'merged',), b'base\n'),
1572
(prefix + (b'merged',), b'left\n'),
1573
(prefix + (b'merged',), b'right\n'),
1574
(prefix + (b'merged',), b'merged\n')
1577
self.assertRaises(RevisionNotPresent,
1578
files.annotate, prefix + ('missing-key',))
1580
def test_check_no_parameters(self):
1581
files = self.get_versionedfiles()
1583
def test_check_progressbar_parameter(self):
1584
"""A progress bar can be supplied because check can be a generator."""
1585
pb = ui.ui_factory.nested_progress_bar()
1586
self.addCleanup(pb.finished)
1587
files = self.get_versionedfiles()
1588
files.check(progress_bar=pb)
1590
def test_check_with_keys_becomes_generator(self):
1591
files = self.get_versionedfiles()
1592
self.get_diamond_files(files)
1594
entries = files.check(keys=keys)
1596
# Texts output should be fulltexts.
1597
self.capture_stream(files, entries, seen.add,
1598
files.get_parent_map(keys), require_fulltext=True)
1599
# All texts should be output.
1600
self.assertEqual(set(keys), seen)
1602
def test_clear_cache(self):
1603
files = self.get_versionedfiles()
1606
def test_construct(self):
1607
"""Each parameterised test can be constructed on a transport."""
1608
files = self.get_versionedfiles()
1610
def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1612
return get_diamond_files(files, self.key_length,
1613
trailing_eol=trailing_eol, nograph=not self.graph,
1614
left_only=left_only, nokeys=nokeys)
1616
def _add_content_nostoresha(self, add_lines):
1617
"""When nostore_sha is supplied using old content raises."""
1618
vf = self.get_versionedfiles()
1619
empty_text = (b'a', [])
1620
sample_text_nl = (b'b', [b"foo\n", b"bar\n"])
1621
sample_text_no_nl = (b'c', [b"foo\n", b"bar"])
1623
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1625
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1628
sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1631
# we now have a copy of all the lines in the vf.
1632
for sha, (version, lines) in zip(
1633
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1634
new_key = self.get_simple_key(version + b"2")
1635
self.assertRaises(errors.ExistingContent,
1636
vf.add_lines, new_key, [], lines,
1638
self.assertRaises(errors.ExistingContent,
1639
vf.add_lines, new_key, [], lines,
1641
# and no new version should have been added.
1642
record = next(vf.get_record_stream([new_key], 'unordered', True))
1643
self.assertEqual('absent', record.storage_kind)
1645
def test_add_lines_nostoresha(self):
1646
self._add_content_nostoresha(add_lines=True)
1648
def test_add_lines_return(self):
1649
files = self.get_versionedfiles()
1650
# save code by using the stock data insertion helper.
1651
adds = self.get_diamond_files(files)
1653
# We can only validate the first 2 elements returned from add_lines.
1655
self.assertEqual(3, len(add))
1656
results.append(add[:2])
1657
if self.key_length == 1:
1659
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1660
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1661
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1662
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1663
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1665
elif self.key_length == 2:
1667
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1668
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1669
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1670
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1671
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1672
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1673
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1674
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1675
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1676
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1679
def test_add_lines_no_key_generates_chk_key(self):
1680
files = self.get_versionedfiles()
1681
# save code by using the stock data insertion helper.
1682
adds = self.get_diamond_files(files, nokeys=True)
1684
# We can only validate the first 2 elements returned from add_lines.
1686
self.assertEqual(3, len(add))
1687
results.append(add[:2])
1688
if self.key_length == 1:
1690
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1691
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1692
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1693
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1694
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1696
# Check the added items got CHK keys.
1698
(b'sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1699
(b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1700
(b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1701
(b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1702
(b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1705
elif self.key_length == 2:
1707
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1708
(b'00e364d235126be43292ab09cb4686cf703ddc17', 7),
1709
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1710
(b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1711
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1712
(b'a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1713
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1714
(b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1715
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1716
(b'ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1718
# Check the added items got CHK keys.
1720
(b'FileA', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1721
(b'FileA', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1722
(b'FileA', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1723
(b'FileA', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1724
(b'FileA', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1725
(b'FileB', b'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1726
(b'FileB', b'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1727
(b'FileB', b'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1728
(b'FileB', b'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1729
(b'FileB', b'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1733
def test_empty_lines(self):
1734
"""Empty files can be stored."""
1735
f = self.get_versionedfiles()
1736
key_a = self.get_simple_key(b'a')
1737
f.add_lines(key_a, [], [])
1738
self.assertEqual(b'',
1739
next(f.get_record_stream([key_a], 'unordered', True
1740
)).get_bytes_as('fulltext'))
1741
key_b = self.get_simple_key(b'b')
1742
f.add_lines(key_b, self.get_parents([key_a]), [])
1743
self.assertEqual(b'',
1744
next(f.get_record_stream([key_b], 'unordered', True
1745
)).get_bytes_as('fulltext'))
1747
def test_newline_only(self):
1748
f = self.get_versionedfiles()
1749
key_a = self.get_simple_key(b'a')
1750
f.add_lines(key_a, [], [b'\n'])
1751
self.assertEqual(b'\n',
1752
next(f.get_record_stream([key_a], 'unordered', True
1753
)).get_bytes_as('fulltext'))
1754
key_b = self.get_simple_key(b'b')
1755
f.add_lines(key_b, self.get_parents([key_a]), [b'\n'])
1756
self.assertEqual(b'\n',
1757
next(f.get_record_stream([key_b], 'unordered', True
1758
)).get_bytes_as('fulltext'))
1760
def test_get_known_graph_ancestry(self):
1761
f = self.get_versionedfiles()
1763
raise TestNotApplicable('ancestry info only relevant with graph.')
1764
key_a = self.get_simple_key(b'a')
1765
key_b = self.get_simple_key(b'b')
1766
key_c = self.get_simple_key(b'c')
1772
f.add_lines(key_a, [], [b'\n'])
1773
f.add_lines(key_b, [key_a], [b'\n'])
1774
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1775
kg = f.get_known_graph_ancestry([key_c])
1776
self.assertIsInstance(kg, _mod_graph.KnownGraph)
1777
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1779
def test_known_graph_with_fallbacks(self):
1780
f = self.get_versionedfiles('files')
1782
raise TestNotApplicable('ancestry info only relevant with graph.')
1783
if getattr(f, 'add_fallback_versioned_files', None) is None:
1784
raise TestNotApplicable("%s doesn't support fallbacks"
1785
% (f.__class__.__name__,))
1786
key_a = self.get_simple_key(b'a')
1787
key_b = self.get_simple_key(b'b')
1788
key_c = self.get_simple_key(b'c')
1789
# A only in fallback
1794
g = self.get_versionedfiles('fallback')
1795
g.add_lines(key_a, [], [b'\n'])
1796
f.add_fallback_versioned_files(g)
1797
f.add_lines(key_b, [key_a], [b'\n'])
1798
f.add_lines(key_c, [key_a, key_b], [b'\n'])
1799
kg = f.get_known_graph_ancestry([key_c])
1800
self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1802
def test_get_record_stream_empty(self):
1803
"""An empty stream can be requested without error."""
1804
f = self.get_versionedfiles()
1805
entries = f.get_record_stream([], 'unordered', False)
1806
self.assertEqual([], list(entries))
1808
def assertValidStorageKind(self, storage_kind):
1809
"""Assert that storage_kind is a valid storage_kind."""
1810
self.assertSubset([storage_kind],
1811
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1812
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1813
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1815
'knit-delta-closure', 'knit-delta-closure-ref',
1816
'groupcompress-block', 'groupcompress-block-ref'])
1818
def capture_stream(self, f, entries, on_seen, parents,
1819
require_fulltext=False):
1820
"""Capture a stream for testing."""
1821
for factory in entries:
1822
on_seen(factory.key)
1823
self.assertValidStorageKind(factory.storage_kind)
1824
if factory.sha1 is not None:
1825
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1827
self.assertEqual(parents[factory.key], factory.parents)
1828
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1830
if require_fulltext:
1831
factory.get_bytes_as('fulltext')
1833
def test_get_record_stream_interface(self):
1834
"""each item in a stream has to provide a regular interface."""
1835
files = self.get_versionedfiles()
1836
self.get_diamond_files(files)
1837
keys, _ = self.get_keys_and_sort_order()
1838
parent_map = files.get_parent_map(keys)
1839
entries = files.get_record_stream(keys, 'unordered', False)
1841
self.capture_stream(files, entries, seen.add, parent_map)
1842
self.assertEqual(set(keys), seen)
1844
def get_keys_and_sort_order(self):
1845
"""Get diamond test keys list, and their sort ordering."""
1846
if self.key_length == 1:
1847
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1848
sort_order = {(b'merged',): 2, (b'left',): 1,
1849
(b'right',): 1, (b'base',): 0}
1852
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1853
(b'FileA', b'base'),
1854
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1855
(b'FileB', b'base'),
1858
(b'FileA', b'merged'): 2, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1859
(b'FileA', b'base'): 0,
1860
(b'FileB', b'merged'): 2, (b'FileB', b'left'): 1, (b'FileB', b'right'): 1,
1861
(b'FileB', b'base'): 0,
1863
return keys, sort_order
1865
def get_keys_and_groupcompress_sort_order(self):
1866
"""Get diamond test keys list, and their groupcompress sort ordering."""
1867
if self.key_length == 1:
1868
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1869
sort_order = {(b'merged',): 0, (b'left',): 1,
1870
(b'right',): 1, (b'base',): 2}
1873
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1874
(b'FileA', b'base'),
1875
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1876
(b'FileB', b'base'),
1879
(b'FileA', b'merged'): 0, (b'FileA', b'left'): 1, (b'FileA', b'right'): 1,
1880
(b'FileA', b'base'): 2,
1881
(b'FileB', b'merged'): 3, (b'FileB', b'left'): 4, (b'FileB', b'right'): 4,
1882
(b'FileB', b'base'): 5,
1884
return keys, sort_order
1886
def test_get_record_stream_interface_ordered(self):
1887
"""each item in a stream has to provide a regular interface."""
1888
files = self.get_versionedfiles()
1889
self.get_diamond_files(files)
1890
keys, sort_order = self.get_keys_and_sort_order()
1891
parent_map = files.get_parent_map(keys)
1892
entries = files.get_record_stream(keys, 'topological', False)
1894
self.capture_stream(files, entries, seen.append, parent_map)
1895
self.assertStreamOrder(sort_order, seen, keys)
1897
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1898
"""each item must be accessible as a fulltext."""
1899
files = self.get_versionedfiles()
1900
self.get_diamond_files(files)
1901
keys, sort_order = self.get_keys_and_sort_order()
1902
parent_map = files.get_parent_map(keys)
1903
entries = files.get_record_stream(keys, 'topological', True)
1905
for factory in entries:
1906
seen.append(factory.key)
1907
self.assertValidStorageKind(factory.storage_kind)
1908
self.assertSubset([factory.sha1],
1909
[None, files.get_sha1s([factory.key])[factory.key]])
1910
self.assertEqual(parent_map[factory.key], factory.parents)
1911
# self.assertEqual(files.get_text(factory.key),
1912
ft_bytes = factory.get_bytes_as('fulltext')
1913
self.assertIsInstance(ft_bytes, bytes)
1914
chunked_bytes = factory.get_bytes_as('chunked')
1915
self.assertEqualDiff(ft_bytes, b''.join(chunked_bytes))
1917
self.assertStreamOrder(sort_order, seen, keys)
1919
def test_get_record_stream_interface_groupcompress(self):
1920
"""each item in a stream has to provide a regular interface."""
1921
files = self.get_versionedfiles()
1922
self.get_diamond_files(files)
1923
keys, sort_order = self.get_keys_and_groupcompress_sort_order()
1924
parent_map = files.get_parent_map(keys)
1925
entries = files.get_record_stream(keys, 'groupcompress', False)
1927
self.capture_stream(files, entries, seen.append, parent_map)
1928
self.assertStreamOrder(sort_order, seen, keys)
1930
def assertStreamOrder(self, sort_order, seen, keys):
1931
self.assertEqual(len(set(seen)), len(keys))
1932
if self.key_length == 1:
1935
lows = {(b'FileA',): 0, (b'FileB',): 0}
1937
self.assertEqual(set(keys), set(seen))
1940
sort_pos = sort_order[key]
1941
self.assertTrue(sort_pos >= lows[key[:-1]],
1942
"Out of order in sorted stream: %r, %r" % (key, seen))
1943
lows[key[:-1]] = sort_pos
1945
def test_get_record_stream_unknown_storage_kind_raises(self):
1946
"""Asking for a storage kind that the stream cannot supply raises."""
1947
files = self.get_versionedfiles()
1948
self.get_diamond_files(files)
1949
if self.key_length == 1:
1950
keys = [(b'merged',), (b'left',), (b'right',), (b'base',)]
1953
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1954
(b'FileA', b'base'),
1955
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1956
(b'FileB', b'base'),
1958
parent_map = files.get_parent_map(keys)
1959
entries = files.get_record_stream(keys, 'unordered', False)
1960
# We track the contents because we should be able to try, fail a
1961
# particular kind and then ask for one that works and continue.
1963
for factory in entries:
1964
seen.add(factory.key)
1965
self.assertValidStorageKind(factory.storage_kind)
1966
if factory.sha1 is not None:
1967
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1969
self.assertEqual(parent_map[factory.key], factory.parents)
1970
# currently no stream emits mpdiff
1971
self.assertRaises(errors.UnavailableRepresentation,
1972
factory.get_bytes_as, 'mpdiff')
1973
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1975
self.assertEqual(set(keys), seen)
1977
def test_get_record_stream_missing_records_are_absent(self):
1978
files = self.get_versionedfiles()
1979
self.get_diamond_files(files)
1980
if self.key_length == 1:
1981
keys = [(b'merged',), (b'left',), (b'right',),
1982
(b'absent',), (b'base',)]
1985
(b'FileA', b'merged'), (b'FileA', b'left'), (b'FileA', b'right'),
1986
(b'FileA', b'absent'), (b'FileA', b'base'),
1987
(b'FileB', b'merged'), (b'FileB', b'left'), (b'FileB', b'right'),
1988
(b'FileB', b'absent'), (b'FileB', b'base'),
1989
(b'absent', b'absent'),
1991
parent_map = files.get_parent_map(keys)
1992
entries = files.get_record_stream(keys, 'unordered', False)
1993
self.assertAbsentRecord(files, keys, parent_map, entries)
1994
entries = files.get_record_stream(keys, 'topological', False)
1995
self.assertAbsentRecord(files, keys, parent_map, entries)
1997
def assertRecordHasContent(self, record, bytes):
1998
"""Assert that record has the bytes bytes."""
1999
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
2000
self.assertEqual(bytes, b''.join(record.get_bytes_as('chunked')))
2002
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
2003
files = self.get_versionedfiles()
2004
key = self.get_simple_key(b'foo')
2005
files.add_lines(key, (), [b'my text\n', b'content'])
2006
stream = files.get_record_stream([key], 'unordered', False)
2007
record = next(stream)
2008
if record.storage_kind in ('chunked', 'fulltext'):
2009
# chunked and fulltext representations are for direct use not wire
2010
# serialisation: check they are able to be used directly. To send
2011
# such records over the wire translation will be needed.
2012
self.assertRecordHasContent(record, b"my text\ncontent")
2014
bytes = [record.get_bytes_as(record.storage_kind)]
2015
network_stream = versionedfile.NetworkRecordStream(bytes).read()
2016
source_record = record
2018
for record in network_stream:
2019
records.append(record)
2020
self.assertEqual(source_record.storage_kind,
2021
record.storage_kind)
2022
self.assertEqual(source_record.parents, record.parents)
2024
source_record.get_bytes_as(source_record.storage_kind),
2025
record.get_bytes_as(record.storage_kind))
2026
self.assertEqual(1, len(records))
2028
def assertStreamMetaEqual(self, records, expected, stream):
2029
"""Assert that streams expected and stream have the same records.
2031
:param records: A list to collect the seen records.
2032
:return: A generator of the records in stream.
2034
# We make assertions during copying to catch things early for easier
2035
# debugging. This must use the iterating zip() from the future.
2036
for record, ref_record in zip(stream, expected):
2037
records.append(record)
2038
self.assertEqual(ref_record.key, record.key)
2039
self.assertEqual(ref_record.storage_kind, record.storage_kind)
2040
self.assertEqual(ref_record.parents, record.parents)
2043
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
2045
"""Convert a stream to a bytes iterator.
2047
:param skipped_records: A list with one element to increment when a
2049
:param full_texts: A dict from key->fulltext representation, for
2050
checking chunked or fulltext stored records.
2051
:param stream: A record_stream.
2052
:return: An iterator over the bytes of each record.
2054
for record in stream:
2055
if record.storage_kind in ('chunked', 'fulltext'):
2056
skipped_records[0] += 1
2057
# check the content is correct for direct use.
2058
self.assertRecordHasContent(record, full_texts[record.key])
2060
yield record.get_bytes_as(record.storage_kind)
2062
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
2063
files = self.get_versionedfiles()
2064
target_files = self.get_versionedfiles('target')
2065
key = self.get_simple_key(b'ft')
2066
key_delta = self.get_simple_key(b'delta')
2067
files.add_lines(key, (), [b'my text\n', b'content'])
2069
delta_parents = (key,)
2072
files.add_lines(key_delta, delta_parents, [
2073
b'different\n', b'content\n'])
2074
local = files.get_record_stream([key, key_delta], 'unordered', False)
2075
ref = files.get_record_stream([key, key_delta], 'unordered', False)
2076
skipped_records = [0]
2078
key: b"my text\ncontent",
2079
key_delta: b"different\ncontent\n",
2081
byte_stream = self.stream_to_bytes_or_skip_counter(
2082
skipped_records, full_texts, local)
2083
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2085
# insert the stream from the network into a versioned files object so we can
2086
# check the content was carried across correctly without doing delta
2088
target_files.insert_record_stream(
2089
self.assertStreamMetaEqual(records, ref, network_stream))
2090
# No duplicates on the wire thank you!
2091
self.assertEqual(2, len(records) + skipped_records[0])
2093
# if any content was copied it all must have all been.
2094
self.assertIdenticalVersionedFile(files, target_files)
2096
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
2097
# copy a delta over the wire
2098
files = self.get_versionedfiles()
2099
target_files = self.get_versionedfiles('target')
2100
key = self.get_simple_key(b'ft')
2101
key_delta = self.get_simple_key(b'delta')
2102
files.add_lines(key, (), [b'my text\n', b'content'])
2104
delta_parents = (key,)
2107
files.add_lines(key_delta, delta_parents, [
2108
b'different\n', b'content\n'])
2109
# Copy the basis text across so we can reconstruct the delta during
2110
# insertion into target.
2111
target_files.insert_record_stream(files.get_record_stream([key],
2112
'unordered', False))
2113
local = files.get_record_stream([key_delta], 'unordered', False)
2114
ref = files.get_record_stream([key_delta], 'unordered', False)
2115
skipped_records = [0]
2117
key_delta: b"different\ncontent\n",
2119
byte_stream = self.stream_to_bytes_or_skip_counter(
2120
skipped_records, full_texts, local)
2121
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2123
# insert the stream from the network into a versioned files object so we can
2124
# check the content was carried across correctly without doing delta
2125
# inspection during check_stream.
2126
target_files.insert_record_stream(
2127
self.assertStreamMetaEqual(records, ref, network_stream))
2128
# No duplicates on the wire thank you!
2129
self.assertEqual(1, len(records) + skipped_records[0])
2131
# if any content was copied it all must have all been
2132
self.assertIdenticalVersionedFile(files, target_files)
2134
def test_get_record_stream_wire_ready_delta_closure_included(self):
2135
# copy a delta over the wire with the ability to get its full text.
2136
files = self.get_versionedfiles()
2137
key = self.get_simple_key(b'ft')
2138
key_delta = self.get_simple_key(b'delta')
2139
files.add_lines(key, (), [b'my text\n', b'content'])
2141
delta_parents = (key,)
2144
files.add_lines(key_delta, delta_parents, [
2145
b'different\n', b'content\n'])
2146
local = files.get_record_stream([key_delta], 'unordered', True)
2147
ref = files.get_record_stream([key_delta], 'unordered', True)
2148
skipped_records = [0]
2150
key_delta: b"different\ncontent\n",
2152
byte_stream = self.stream_to_bytes_or_skip_counter(
2153
skipped_records, full_texts, local)
2154
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2156
# insert the stream from the network into a versioned files object so we can
2157
# check the content was carried across correctly without doing delta
2158
# inspection during check_stream.
2159
for record in self.assertStreamMetaEqual(records, ref, network_stream):
2160
# we have to be able to get the full text out:
2161
self.assertRecordHasContent(record, full_texts[record.key])
2162
# No duplicates on the wire thank you!
2163
self.assertEqual(1, len(records) + skipped_records[0])
2165
def assertAbsentRecord(self, files, keys, parents, entries):
2166
"""Helper for test_get_record_stream_missing_records_are_absent."""
2168
for factory in entries:
2169
seen.add(factory.key)
2170
if factory.key[-1] == b'absent':
2171
self.assertEqual('absent', factory.storage_kind)
2172
self.assertEqual(None, factory.sha1)
2173
self.assertEqual(None, factory.parents)
2175
self.assertValidStorageKind(factory.storage_kind)
2176
if factory.sha1 is not None:
2177
sha1 = files.get_sha1s([factory.key])[factory.key]
2178
self.assertEqual(sha1, factory.sha1)
2179
self.assertEqual(parents[factory.key], factory.parents)
2180
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2182
self.assertEqual(set(keys), seen)
2184
def test_filter_absent_records(self):
2185
"""Requested missing records can be filter trivially."""
2186
files = self.get_versionedfiles()
2187
self.get_diamond_files(files)
2188
keys, _ = self.get_keys_and_sort_order()
2189
parent_map = files.get_parent_map(keys)
2190
# Add an absent record in the middle of the present keys. (We don't ask
2191
# for just absent keys to ensure that content before and after the
2192
# absent keys is still delivered).
2193
present_keys = list(keys)
2194
if self.key_length == 1:
2195
keys.insert(2, (b'extra',))
2197
keys.insert(2, (b'extra', b'extra'))
2198
entries = files.get_record_stream(keys, 'unordered', False)
2200
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2202
self.assertEqual(set(present_keys), seen)
2204
def get_mapper(self):
2205
"""Get a mapper suitable for the key length of the test interface."""
2206
if self.key_length == 1:
2207
return ConstantMapper('source')
2209
return HashEscapedPrefixMapper()
2211
def get_parents(self, parents):
2212
"""Get parents, taking self.graph into consideration."""
2218
def test_get_annotator(self):
2219
files = self.get_versionedfiles()
2220
self.get_diamond_files(files)
2221
origin_key = self.get_simple_key(b'origin')
2222
base_key = self.get_simple_key(b'base')
2223
left_key = self.get_simple_key(b'left')
2224
right_key = self.get_simple_key(b'right')
2225
merged_key = self.get_simple_key(b'merged')
2226
# annotator = files.get_annotator()
2227
# introduced full text
2228
origins, lines = files.get_annotator().annotate(origin_key)
2229
self.assertEqual([(origin_key,)], origins)
2230
self.assertEqual([b'origin\n'], lines)
2232
origins, lines = files.get_annotator().annotate(base_key)
2233
self.assertEqual([(base_key,)], origins)
2235
origins, lines = files.get_annotator().annotate(merged_key)
2244
# Without a graph everything is new.
2251
self.assertRaises(RevisionNotPresent,
2252
files.get_annotator().annotate, self.get_simple_key(b'missing-key'))
2254
def test_get_parent_map(self):
2255
files = self.get_versionedfiles()
2256
if self.key_length == 1:
2258
((b'r0',), self.get_parents(())),
2259
((b'r1',), self.get_parents(((b'r0',),))),
2260
((b'r2',), self.get_parents(())),
2261
((b'r3',), self.get_parents(())),
2262
((b'm',), self.get_parents(((b'r0',), (b'r1',), (b'r2',), (b'r3',)))),
2266
((b'FileA', b'r0'), self.get_parents(())),
2267
((b'FileA', b'r1'), self.get_parents(((b'FileA', b'r0'),))),
2268
((b'FileA', b'r2'), self.get_parents(())),
2269
((b'FileA', b'r3'), self.get_parents(())),
2270
((b'FileA', b'm'), self.get_parents(((b'FileA', b'r0'),
2271
(b'FileA', b'r1'), (b'FileA', b'r2'), (b'FileA', b'r3')))),
2273
for key, parents in parent_details:
2274
files.add_lines(key, parents, [])
2275
# immediately after adding it should be queryable.
2276
self.assertEqual({key: parents}, files.get_parent_map([key]))
2277
# We can ask for an empty set
2278
self.assertEqual({}, files.get_parent_map([]))
2279
# We can ask for many keys
2280
all_parents = dict(parent_details)
2281
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2282
# Absent keys are just not included in the result.
2283
keys = list(all_parents.keys())
2284
if self.key_length == 1:
2285
keys.insert(1, (b'missing',))
2287
keys.insert(1, (b'missing', b'missing'))
2288
# Absent keys are just ignored
2289
self.assertEqual(all_parents, files.get_parent_map(keys))
2291
def test_get_sha1s(self):
2292
files = self.get_versionedfiles()
2293
self.get_diamond_files(files)
2294
if self.key_length == 1:
2295
keys = [(b'base',), (b'origin',), (b'left',),
2296
(b'merged',), (b'right',)]
2298
# ask for shas from different prefixes.
2300
(b'FileA', b'base'), (b'FileB', b'origin'), (b'FileA', b'left'),
2301
(b'FileA', b'merged'), (b'FileB', b'right'),
2304
keys[0]: b'51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2305
keys[1]: b'00e364d235126be43292ab09cb4686cf703ddc17',
2306
keys[2]: b'a8478686da38e370e32e42e8a0c220e33ee9132f',
2307
keys[3]: b'ed8bce375198ea62444dc71952b22cfc2b09226d',
2308
keys[4]: b'9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2310
files.get_sha1s(keys))
2312
def test_insert_record_stream_empty(self):
2313
"""Inserting an empty record stream should work."""
2314
files = self.get_versionedfiles()
2315
files.insert_record_stream([])
2317
def assertIdenticalVersionedFile(self, expected, actual):
2318
"""Assert that left and right have the same contents."""
2319
self.assertEqual(set(actual.keys()), set(expected.keys()))
2320
actual_parents = actual.get_parent_map(actual.keys())
2323
actual_parents, expected.get_parent_map(expected.keys()))
2325
for key, parents in actual_parents.items():
2326
self.assertEqual(None, parents)
2327
for key in actual.keys():
2328
actual_text = next(actual.get_record_stream(
2329
[key], 'unordered', True)).get_bytes_as('fulltext')
2330
expected_text = next(expected.get_record_stream(
2331
[key], 'unordered', True)).get_bytes_as('fulltext')
2332
self.assertEqual(actual_text, expected_text)
2334
def test_insert_record_stream_fulltexts(self):
2335
"""Any file should accept a stream of fulltexts."""
2336
files = self.get_versionedfiles()
2337
mapper = self.get_mapper()
2338
source_transport = self.get_transport('source')
2339
source_transport.mkdir('.')
2340
# weaves always output fulltexts.
2341
source = make_versioned_files_factory(WeaveFile, mapper)(
2343
self.get_diamond_files(source, trailing_eol=False)
2344
stream = source.get_record_stream(source.keys(), 'topological',
2346
files.insert_record_stream(stream)
2347
self.assertIdenticalVersionedFile(source, files)
2349
def test_insert_record_stream_fulltexts_noeol(self):
2350
"""Any file should accept a stream of fulltexts."""
2351
files = self.get_versionedfiles()
2352
mapper = self.get_mapper()
2353
source_transport = self.get_transport('source')
2354
source_transport.mkdir('.')
2355
# weaves always output fulltexts.
2356
source = make_versioned_files_factory(WeaveFile, mapper)(
2358
self.get_diamond_files(source, trailing_eol=False)
2359
stream = source.get_record_stream(source.keys(), 'topological',
2361
files.insert_record_stream(stream)
2362
self.assertIdenticalVersionedFile(source, files)
2364
def test_insert_record_stream_annotated_knits(self):
2365
"""Any file should accept a stream from plain knits."""
2366
files = self.get_versionedfiles()
2367
mapper = self.get_mapper()
2368
source_transport = self.get_transport('source')
2369
source_transport.mkdir('.')
2370
source = make_file_factory(True, mapper)(source_transport)
2371
self.get_diamond_files(source)
2372
stream = source.get_record_stream(source.keys(), 'topological',
2374
files.insert_record_stream(stream)
2375
self.assertIdenticalVersionedFile(source, files)
2377
def test_insert_record_stream_annotated_knits_noeol(self):
2378
"""Any file should accept a stream from plain knits."""
2379
files = self.get_versionedfiles()
2380
mapper = self.get_mapper()
2381
source_transport = self.get_transport('source')
2382
source_transport.mkdir('.')
2383
source = make_file_factory(True, mapper)(source_transport)
2384
self.get_diamond_files(source, trailing_eol=False)
2385
stream = source.get_record_stream(source.keys(), 'topological',
2387
files.insert_record_stream(stream)
2388
self.assertIdenticalVersionedFile(source, files)
2390
def test_insert_record_stream_plain_knits(self):
2391
"""Any file should accept a stream from plain knits."""
2392
files = self.get_versionedfiles()
2393
mapper = self.get_mapper()
2394
source_transport = self.get_transport('source')
2395
source_transport.mkdir('.')
2396
source = make_file_factory(False, mapper)(source_transport)
2397
self.get_diamond_files(source)
2398
stream = source.get_record_stream(source.keys(), 'topological',
2400
files.insert_record_stream(stream)
2401
self.assertIdenticalVersionedFile(source, files)
2403
def test_insert_record_stream_plain_knits_noeol(self):
2404
"""Any file should accept a stream from plain knits."""
2405
files = self.get_versionedfiles()
2406
mapper = self.get_mapper()
2407
source_transport = self.get_transport('source')
2408
source_transport.mkdir('.')
2409
source = make_file_factory(False, mapper)(source_transport)
2410
self.get_diamond_files(source, trailing_eol=False)
2411
stream = source.get_record_stream(source.keys(), 'topological',
2413
files.insert_record_stream(stream)
2414
self.assertIdenticalVersionedFile(source, files)
2416
def test_insert_record_stream_existing_keys(self):
2417
"""Inserting keys already in a file should not error."""
2418
files = self.get_versionedfiles()
2419
source = self.get_versionedfiles('source')
2420
self.get_diamond_files(source)
2421
# insert some keys into f.
2422
self.get_diamond_files(files, left_only=True)
2423
stream = source.get_record_stream(source.keys(), 'topological',
2425
files.insert_record_stream(stream)
2426
self.assertIdenticalVersionedFile(source, files)
2428
def test_insert_record_stream_missing_keys(self):
2429
"""Inserting a stream with absent keys should raise an error."""
2430
files = self.get_versionedfiles()
2431
source = self.get_versionedfiles('source')
2432
stream = source.get_record_stream([(b'missing',) * self.key_length],
2433
'topological', False)
2434
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2437
def test_insert_record_stream_out_of_order(self):
2438
"""An out of order stream can either error or work."""
2439
files = self.get_versionedfiles()
2440
source = self.get_versionedfiles('source')
2441
self.get_diamond_files(source)
2442
if self.key_length == 1:
2443
origin_keys = [(b'origin',)]
2444
end_keys = [(b'merged',), (b'left',)]
2445
start_keys = [(b'right',), (b'base',)]
2447
origin_keys = [(b'FileA', b'origin'), (b'FileB', b'origin')]
2448
end_keys = [(b'FileA', b'merged',), (b'FileA', b'left',),
2449
(b'FileB', b'merged',), (b'FileB', b'left',)]
2450
start_keys = [(b'FileA', b'right',), (b'FileA', b'base',),
2451
(b'FileB', b'right',), (b'FileB', b'base',)]
2452
origin_entries = source.get_record_stream(
2453
origin_keys, 'unordered', False)
2454
end_entries = source.get_record_stream(end_keys, 'topological', False)
2455
start_entries = source.get_record_stream(
2456
start_keys, 'topological', False)
2457
entries = itertools.chain(origin_entries, end_entries, start_entries)
2459
files.insert_record_stream(entries)
2460
except RevisionNotPresent:
2461
# Must not have corrupted the file.
2464
self.assertIdenticalVersionedFile(source, files)
2466
def test_insert_record_stream_long_parent_chain_out_of_order(self):
2467
"""An out of order stream can either error or work."""
2469
raise TestNotApplicable('ancestry info only relevant with graph.')
2470
# Create a reasonably long chain of records based on each other, where
2471
# most will be deltas.
2472
source = self.get_versionedfiles('source')
2475
content = [(b'same same %d\n' % n) for n in range(500)]
2476
letters = b'abcdefghijklmnopqrstuvwxyz'
2477
for i in range(len(letters)):
2478
letter = letters[i:i + 1]
2479
key = (b'key-' + letter,)
2480
if self.key_length == 2:
2481
key = (b'prefix',) + key
2482
content.append(b'content for ' + letter + b'\n')
2483
source.add_lines(key, parents, content)
2486
# Create a stream of these records, excluding the first record that the
2487
# rest ultimately depend upon, and insert it into a new vf.
2489
for key in reversed(keys):
2490
streams.append(source.get_record_stream([key], 'unordered', False))
2491
deltas = itertools.chain.from_iterable(streams[:-1])
2492
files = self.get_versionedfiles()
2494
files.insert_record_stream(deltas)
2495
except RevisionNotPresent:
2496
# Must not have corrupted the file.
2499
# Must only report either just the first key as a missing parent,
2500
# no key as missing (for nodelta scenarios).
2501
missing = set(files.get_missing_compression_parent_keys())
2502
missing.discard(keys[0])
2503
self.assertEqual(set(), missing)
2505
def get_knit_delta_source(self):
2506
"""Get a source that can produce a stream with knit delta records,
2507
regardless of this test's scenario.
2509
mapper = self.get_mapper()
2510
source_transport = self.get_transport('source')
2511
source_transport.mkdir('.')
2512
source = make_file_factory(False, mapper)(source_transport)
2513
get_diamond_files(source, self.key_length, trailing_eol=True,
2514
nograph=False, left_only=False)
2517
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2518
"""Insertion where a needed basis is not included notifies the caller
2519
of the missing basis. In the meantime a record missing its basis is
2522
source = self.get_knit_delta_source()
2523
keys = [self.get_simple_key(b'origin'), self.get_simple_key(b'merged')]
2524
entries = source.get_record_stream(keys, 'unordered', False)
2525
files = self.get_versionedfiles()
2526
if self.support_partial_insertion:
2527
self.assertEqual([],
2528
list(files.get_missing_compression_parent_keys()))
2529
files.insert_record_stream(entries)
2530
missing_bases = files.get_missing_compression_parent_keys()
2531
self.assertEqual({self.get_simple_key(b'left')},
2533
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2536
errors.RevisionNotPresent, files.insert_record_stream, entries)
2539
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2540
"""Insertion where a needed basis is not included notifies the caller
2541
of the missing basis. That basis can be added in a second
2542
insert_record_stream call that does not need to repeat records present
2543
in the previous stream. The record(s) that required that basis are
2544
fully inserted once their basis is no longer missing.
2546
if not self.support_partial_insertion:
2547
raise TestNotApplicable(
2548
'versioned file scenario does not support partial insertion')
2549
source = self.get_knit_delta_source()
2550
entries = source.get_record_stream([self.get_simple_key(b'origin'),
2551
self.get_simple_key(b'merged')], 'unordered', False)
2552
files = self.get_versionedfiles()
2553
files.insert_record_stream(entries)
2554
missing_bases = files.get_missing_compression_parent_keys()
2555
self.assertEqual({self.get_simple_key(b'left')},
2557
# 'merged' is inserted (although a commit of a write group involving
2558
# this versionedfiles would fail).
2559
merged_key = self.get_simple_key(b'merged')
2561
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2562
# Add the full delta closure of the missing records
2563
missing_entries = source.get_record_stream(
2564
missing_bases, 'unordered', True)
2565
files.insert_record_stream(missing_entries)
2566
# Now 'merged' is fully inserted (and a commit would succeed).
2567
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2569
[merged_key], list(files.get_parent_map([merged_key]).keys()))
2572
def test_iter_lines_added_or_present_in_keys(self):
2573
# test that we get at least an equalset of the lines added by
2574
# versions in the store.
2575
# the ordering here is to make a tree so that dumb searches have
2576
# more changes to muck up.
2578
class InstrumentedProgress(progress.ProgressTask):
2581
progress.ProgressTask.__init__(self)
2584
def update(self, msg=None, current=None, total=None):
2585
self.updates.append((msg, current, total))
2587
files = self.get_versionedfiles()
2588
# add a base to get included
2589
files.add_lines(self.get_simple_key(b'base'), (), [b'base\n'])
2590
# add a ancestor to be included on one side
2591
files.add_lines(self.get_simple_key(
2592
b'lancestor'), (), [b'lancestor\n'])
2593
# add a ancestor to be included on the other side
2594
files.add_lines(self.get_simple_key(b'rancestor'),
2595
self.get_parents([self.get_simple_key(b'base')]), [b'rancestor\n'])
2596
# add a child of rancestor with no eofile-nl
2597
files.add_lines(self.get_simple_key(b'child'),
2598
self.get_parents([self.get_simple_key(b'rancestor')]),
2599
[b'base\n', b'child\n'])
2600
# add a child of lancestor and base to join the two roots
2601
files.add_lines(self.get_simple_key(b'otherchild'),
2602
self.get_parents([self.get_simple_key(b'lancestor'),
2603
self.get_simple_key(b'base')]),
2604
[b'base\n', b'lancestor\n', b'otherchild\n'])
2606
def iter_with_keys(keys, expected):
2607
# now we need to see what lines are returned, and how often.
2609
progress = InstrumentedProgress()
2610
# iterate over the lines
2611
for line in files.iter_lines_added_or_present_in_keys(keys,
2613
lines.setdefault(line, 0)
2615
if [] != progress.updates:
2616
self.assertEqual(expected, progress.updates)
2618
lines = iter_with_keys(
2619
[self.get_simple_key(b'child'),
2620
self.get_simple_key(b'otherchild')],
2621
[('Walking content', 0, 2),
2622
('Walking content', 1, 2),
2623
('Walking content', 2, 2)])
2624
# we must see child and otherchild
2625
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2627
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2628
# we dont care if we got more than that.
2631
lines = iter_with_keys(files.keys(),
2632
[('Walking content', 0, 5),
2633
('Walking content', 1, 5),
2634
('Walking content', 2, 5),
2635
('Walking content', 3, 5),
2636
('Walking content', 4, 5),
2637
('Walking content', 5, 5)])
2638
# all lines must be seen at least once
2639
self.assertTrue(lines[(b'base\n', self.get_simple_key(b'base'))] > 0)
2641
lines[(b'lancestor\n', self.get_simple_key(b'lancestor'))] > 0)
2643
lines[(b'rancestor\n', self.get_simple_key(b'rancestor'))] > 0)
2644
self.assertTrue(lines[(b'child\n', self.get_simple_key(b'child'))] > 0)
2646
lines[(b'otherchild\n', self.get_simple_key(b'otherchild'))] > 0)
2648
def test_make_mpdiffs(self):
2649
from breezy import multiparent
2650
files = self.get_versionedfiles('source')
2651
# add texts that should trip the knit maximum delta chain threshold
2652
# as well as doing parallel chains of data in knits.
2653
# this is done by two chains of 25 insertions
2654
files.add_lines(self.get_simple_key(b'base'), [], [b'line\n'])
2655
files.add_lines(self.get_simple_key(b'noeol'),
2656
self.get_parents([self.get_simple_key(b'base')]), [b'line'])
2657
# detailed eol tests:
2658
# shared last line with parent no-eol
2659
files.add_lines(self.get_simple_key(b'noeolsecond'),
2660
self.get_parents([self.get_simple_key(b'noeol')]),
2661
[b'line\n', b'line'])
2662
# differing last line with parent, both no-eol
2663
files.add_lines(self.get_simple_key(b'noeolnotshared'),
2665
[self.get_simple_key(b'noeolsecond')]),
2666
[b'line\n', b'phone'])
2667
# add eol following a noneol parent, change content
2668
files.add_lines(self.get_simple_key(b'eol'),
2669
self.get_parents([self.get_simple_key(b'noeol')]), [b'phone\n'])
2670
# add eol following a noneol parent, no change content
2671
files.add_lines(self.get_simple_key(b'eolline'),
2672
self.get_parents([self.get_simple_key(b'noeol')]), [b'line\n'])
2673
# noeol with no parents:
2674
files.add_lines(self.get_simple_key(b'noeolbase'), [], [b'line'])
2675
# noeol preceeding its leftmost parent in the output:
2676
# this is done by making it a merge of two parents with no common
2677
# anestry: noeolbase and noeol with the
2678
# later-inserted parent the leftmost.
2679
files.add_lines(self.get_simple_key(b'eolbeforefirstparent'),
2680
self.get_parents([self.get_simple_key(b'noeolbase'),
2681
self.get_simple_key(b'noeol')]),
2683
# two identical eol texts
2684
files.add_lines(self.get_simple_key(b'noeoldup'),
2685
self.get_parents([self.get_simple_key(b'noeol')]), [b'line'])
2686
next_parent = self.get_simple_key(b'base')
2687
text_name = b'chain1-'
2689
sha1s = {0: b'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2690
1: b'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2691
2: b'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2692
3: b'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2693
4: b'e28a5510be25ba84d31121cff00956f9970ae6f6',
2694
5: b'd63ec0ce22e11dcf65a931b69255d3ac747a318d',
2695
6: b'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2696
7: b'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2697
8: b'779e9a0b28f9f832528d4b21e17e168c67697272',
2698
9: b'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2699
10: b'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2700
11: b'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2701
12: b'31a2286267f24d8bedaa43355f8ad7129509ea85',
2702
13: b'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2703
14: b'2c4b1736566b8ca6051e668de68650686a3922f2',
2704
15: b'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2705
16: b'b0d2e18d3559a00580f6b49804c23fea500feab3',
2706
17: b'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2707
18: b'5cf64a3459ae28efa60239e44b20312d25b253f3',
2708
19: b'1ebed371807ba5935958ad0884595126e8c4e823',
2709
20: b'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2710
21: b'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2711
22: b'd8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2712
23: b'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2713
24: b'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2714
25: b'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2716
for depth in range(26):
2717
new_version = self.get_simple_key(text_name + b'%d' % depth)
2718
text = text + [b'line\n']
2719
files.add_lines(new_version, self.get_parents([next_parent]), text)
2720
next_parent = new_version
2721
next_parent = self.get_simple_key(b'base')
2722
text_name = b'chain2-'
2724
for depth in range(26):
2725
new_version = self.get_simple_key(text_name + b'%d' % depth)
2726
text = text + [b'line\n']
2727
files.add_lines(new_version, self.get_parents([next_parent]), text)
2728
next_parent = new_version
2729
target = self.get_versionedfiles('target')
2730
for key in multiparent.topo_iter_keys(files, files.keys()):
2731
mpdiff = files.make_mpdiffs([key])[0]
2732
parents = files.get_parent_map([key])[key] or []
2734
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2735
self.assertEqualDiff(
2736
next(files.get_record_stream([key], 'unordered',
2737
True)).get_bytes_as('fulltext'),
2738
next(target.get_record_stream([key], 'unordered',
2739
True)).get_bytes_as('fulltext')
2742
def test_keys(self):
2743
# While use is discouraged, versions() is still needed by aspects of
2745
files = self.get_versionedfiles()
2746
self.assertEqual(set(), set(files.keys()))
2747
if self.key_length == 1:
2750
key = (b'foo', b'bar',)
2751
files.add_lines(key, (), [])
2752
self.assertEqual({key}, set(files.keys()))
2755
class VirtualVersionedFilesTests(TestCase):
2756
"""Basic tests for the VirtualVersionedFiles implementations."""
2758
def _get_parent_map(self, keys):
2761
if k in self._parent_map:
2762
ret[k] = self._parent_map[k]
2766
super(VirtualVersionedFilesTests, self).setUp()
2768
self._parent_map = {}
2769
self.texts = VirtualVersionedFiles(self._get_parent_map,
2772
def test_add_lines(self):
2773
self.assertRaises(NotImplementedError,
2774
self.texts.add_lines, b"foo", [], [])
2776
def test_add_mpdiffs(self):
2777
self.assertRaises(NotImplementedError,
2778
self.texts.add_mpdiffs, [])
2780
def test_check_noerrors(self):
2783
def test_insert_record_stream(self):
2784
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2787
def test_get_sha1s_nonexistent(self):
2788
self.assertEqual({}, self.texts.get_sha1s([(b"NONEXISTENT",)]))
2790
def test_get_sha1s(self):
2791
self._lines[b"key"] = [b"dataline1", b"dataline2"]
2792
self.assertEqual({(b"key",): osutils.sha_strings(self._lines[b"key"])},
2793
self.texts.get_sha1s([(b"key",)]))
2795
def test_get_parent_map(self):
2796
self._parent_map = {b"G": (b"A", b"B")}
2797
self.assertEqual({(b"G",): ((b"A",), (b"B",))},
2798
self.texts.get_parent_map([(b"G",), (b"L",)]))
2800
def test_get_record_stream(self):
2801
self._lines[b"A"] = [b"FOO", b"BAR"]
2802
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2804
self.assertEqual("chunked", record.storage_kind)
2805
self.assertEqual(b"FOOBAR", record.get_bytes_as("fulltext"))
2806
self.assertEqual([b"FOO", b"BAR"], record.get_bytes_as("chunked"))
2808
def test_get_record_stream_absent(self):
2809
it = self.texts.get_record_stream([(b"A",)], "unordered", True)
2811
self.assertEqual("absent", record.storage_kind)
2813
def test_iter_lines_added_or_present_in_keys(self):
2814
self._lines[b"A"] = [b"FOO", b"BAR"]
2815
self._lines[b"B"] = [b"HEY"]
2816
self._lines[b"C"] = [b"Alberta"]
2817
it = self.texts.iter_lines_added_or_present_in_keys([(b"A",), (b"B",)])
2818
self.assertEqual(sorted([(b"FOO", b"A"), (b"BAR", b"A"), (b"HEY", b"B")]),
2822
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2824
def get_ordering_vf(self, key_priority):
2825
builder = self.make_branch_builder('test')
2826
builder.start_series()
2827
builder.build_snapshot(None, [
2828
('add', ('', b'TREE_ROOT', 'directory', None))],
2830
builder.build_snapshot([b'A'], [], revision_id=b'B')
2831
builder.build_snapshot([b'B'], [], revision_id=b'C')
2832
builder.build_snapshot([b'C'], [], revision_id=b'D')
2833
builder.finish_series()
2834
b = builder.get_branch()
2836
self.addCleanup(b.unlock)
2837
vf = b.repository.inventories
2838
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2840
def test_get_empty(self):
2841
vf = self.get_ordering_vf({})
2842
self.assertEqual([], vf.calls)
2844
def test_get_record_stream_topological(self):
2845
vf = self.get_ordering_vf(
2846
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2847
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2848
keys = [r.key for r in vf.get_record_stream(request_keys,
2849
'topological', False)]
2850
# We should have gotten the keys in topological order
2851
self.assertEqual([(b'A',), (b'B',), (b'C',), (b'D',)], keys)
2852
# And recorded that the request was made
2853
self.assertEqual([('get_record_stream', request_keys, 'topological',
2856
def test_get_record_stream_ordered(self):
2857
vf = self.get_ordering_vf(
2858
{(b'A',): 3, (b'B',): 2, (b'C',): 4, (b'D',): 1})
2859
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2860
keys = [r.key for r in vf.get_record_stream(request_keys,
2861
'unordered', False)]
2862
# They should be returned based on their priority
2863
self.assertEqual([(b'D',), (b'B',), (b'A',), (b'C',)], keys)
2864
# And the request recorded
2865
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2868
def test_get_record_stream_implicit_order(self):
2869
vf = self.get_ordering_vf({(b'B',): 2, (b'D',): 1})
2870
request_keys = [(b'B',), (b'C',), (b'D',), (b'A',)]
2871
keys = [r.key for r in vf.get_record_stream(request_keys,
2872
'unordered', False)]
2873
# A and C are not in the map, so they get sorted to the front. A comes
2874
# before C alphabetically, so it comes back first
2875
self.assertEqual([(b'A',), (b'C',), (b'D',), (b'B',)], keys)
2876
# And the request recorded
2877
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
breezy/tests/per_versionedfile.py
