1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Knit versionedfile implementation.
19
A knit is a versioned file implementation that supports efficient append only
23
lifeless: the data file is made up of "delta records". each delta record has a delta header
24
that contains; (1) a version id, (2) the size of the delta (in lines), and (3) the digest of
25
the -expanded data- (ie, the delta applied to the parent). the delta also ends with a
26
end-marker; simply "end VERSION"
28
delta can be line or full contents.a
29
... the 8's there are the index number of the annotation.
30
version robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad 7 c7d23b2a5bd6ca00e8e266cec0ec228158ee9f9e
34
8 e.set('executable', 'yes')
36
8 if elt.get('executable') == 'yes':
37
8 ie.executable = True
38
end robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad
42
09:33 < jrydberg> lifeless: each index is made up of a tuple of; version id, options, position, size, parents
43
09:33 < jrydberg> lifeless: the parents are currently dictionary compressed
44
09:33 < jrydberg> lifeless: (meaning it currently does not support ghosts)
45
09:33 < lifeless> right
46
09:33 < jrydberg> lifeless: the position and size is the range in the data file
49
so the index sequence is the dictionary compressed sequence number used
50
in the deltas to provide line annotation
55
# 10:16 < lifeless> make partial index writes safe
56
# 10:16 < lifeless> implement 'knit.check()' like weave.check()
57
# 10:17 < lifeless> record known ghosts so we can detect when they are filled in rather than the current 'reweave
59
# move sha1 out of the content so that join is faster at verifying parents
60
# record content length ?
63
from cStringIO import StringIO
64
from itertools import izip, chain
69
from bzrlib.lazy_import import lazy_import
70
lazy_import(globals(), """
90
from bzrlib.errors import (
98
RevisionAlreadyPresent,
101
from bzrlib.osutils import (
108
from bzrlib.versionedfile import (
109
AbsentContentFactory,
113
ChunkedContentFactory,
119
# TODO: Split out code specific to this format into an associated object.
121
# TODO: Can we put in some kind of value to check that the index and data
122
# files belong together?
124
# TODO: accommodate binaries, perhaps by storing a byte count
126
# TODO: function to check whole file
128
# TODO: atomically append data, then measure backwards from the cursor
129
# position after writing to work out where it was located. we may need to
130
# bypass python file buffering.
132
DATA_SUFFIX = '.knit'
133
INDEX_SUFFIX = '.kndx'
136
class KnitAdapter(object):
137
"""Base class for knit record adaption."""
139
def __init__(self, basis_vf):
140
"""Create an adapter which accesses full texts from basis_vf.
142
:param basis_vf: A versioned file to access basis texts of deltas from.
143
May be None for adapters that do not need to access basis texts.
145
self._data = KnitVersionedFiles(None, None)
146
self._annotate_factory = KnitAnnotateFactory()
147
self._plain_factory = KnitPlainFactory()
148
self._basis_vf = basis_vf
151
class FTAnnotatedToUnannotated(KnitAdapter):
152
"""An adapter from FT annotated knits to unannotated ones."""
154
def get_bytes(self, factory, annotated_compressed_bytes):
156
self._data._parse_record_unchecked(annotated_compressed_bytes)
157
content = self._annotate_factory.parse_fulltext(contents, rec[1])
158
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
162
class DeltaAnnotatedToUnannotated(KnitAdapter):
163
"""An adapter for deltas from annotated to unannotated."""
165
def get_bytes(self, factory, annotated_compressed_bytes):
167
self._data._parse_record_unchecked(annotated_compressed_bytes)
168
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
170
contents = self._plain_factory.lower_line_delta(delta)
171
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
175
class FTAnnotatedToFullText(KnitAdapter):
176
"""An adapter from FT annotated knits to unannotated ones."""
178
def get_bytes(self, factory, annotated_compressed_bytes):
180
self._data._parse_record_unchecked(annotated_compressed_bytes)
181
content, delta = self._annotate_factory.parse_record(factory.key[-1],
182
contents, factory._build_details, None)
183
return ''.join(content.text())
186
class DeltaAnnotatedToFullText(KnitAdapter):
187
"""An adapter for deltas from annotated to unannotated."""
189
def get_bytes(self, factory, annotated_compressed_bytes):
191
self._data._parse_record_unchecked(annotated_compressed_bytes)
192
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
194
compression_parent = factory.parents[0]
195
basis_entry = self._basis_vf.get_record_stream(
196
[compression_parent], 'unordered', True).next()
197
if basis_entry.storage_kind == 'absent':
198
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
199
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
200
# Manually apply the delta because we have one annotated content and
202
basis_content = PlainKnitContent(basis_lines, compression_parent)
203
basis_content.apply_delta(delta, rec[1])
204
basis_content._should_strip_eol = factory._build_details[1]
205
return ''.join(basis_content.text())
208
class FTPlainToFullText(KnitAdapter):
209
"""An adapter from FT plain knits to unannotated ones."""
211
def get_bytes(self, factory, compressed_bytes):
213
self._data._parse_record_unchecked(compressed_bytes)
214
content, delta = self._plain_factory.parse_record(factory.key[-1],
215
contents, factory._build_details, None)
216
return ''.join(content.text())
219
class DeltaPlainToFullText(KnitAdapter):
220
"""An adapter for deltas from annotated to unannotated."""
222
def get_bytes(self, factory, compressed_bytes):
224
self._data._parse_record_unchecked(compressed_bytes)
225
delta = self._plain_factory.parse_line_delta(contents, rec[1])
226
compression_parent = factory.parents[0]
227
# XXX: string splitting overhead.
228
basis_entry = self._basis_vf.get_record_stream(
229
[compression_parent], 'unordered', True).next()
230
if basis_entry.storage_kind == 'absent':
231
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
232
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
233
basis_content = PlainKnitContent(basis_lines, compression_parent)
234
# Manually apply the delta because we have one annotated content and
236
content, _ = self._plain_factory.parse_record(rec[1], contents,
237
factory._build_details, basis_content)
238
return ''.join(content.text())
241
class KnitContentFactory(ContentFactory):
242
"""Content factory for streaming from knits.
244
:seealso ContentFactory:
247
def __init__(self, key, parents, build_details, sha1, raw_record,
248
annotated, knit=None):
249
"""Create a KnitContentFactory for key.
252
:param parents: The parents.
253
:param build_details: The build details as returned from
255
:param sha1: The sha1 expected from the full text of this object.
256
:param raw_record: The bytes of the knit data from disk.
257
:param annotated: True if the raw data is annotated.
259
ContentFactory.__init__(self)
262
self.parents = parents
263
if build_details[0] == 'line-delta':
268
annotated_kind = 'annotated-'
271
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
272
self._raw_record = raw_record
273
self._build_details = build_details
276
def get_bytes_as(self, storage_kind):
277
if storage_kind == self.storage_kind:
278
return self._raw_record
279
if self._knit is not None:
280
if storage_kind == 'chunked':
281
return self._knit.get_lines(self.key[0])
282
elif storage_kind == 'fulltext':
283
return self._knit.get_text(self.key[0])
284
raise errors.UnavailableRepresentation(self.key, storage_kind,
288
class KnitContent(object):
289
"""Content of a knit version to which deltas can be applied.
291
This is always stored in memory as a list of lines with \n at the end,
292
plus a flag saying if the final ending is really there or not, because that
293
corresponds to the on-disk knit representation.
297
self._should_strip_eol = False
299
def apply_delta(self, delta, new_version_id):
300
"""Apply delta to this object to become new_version_id."""
301
raise NotImplementedError(self.apply_delta)
303
def line_delta_iter(self, new_lines):
304
"""Generate line-based delta from this content to new_lines."""
305
new_texts = new_lines.text()
306
old_texts = self.text()
307
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
308
for tag, i1, i2, j1, j2 in s.get_opcodes():
311
# ofrom, oto, length, data
312
yield i1, i2, j2 - j1, new_lines._lines[j1:j2]
314
def line_delta(self, new_lines):
315
return list(self.line_delta_iter(new_lines))
318
def get_line_delta_blocks(knit_delta, source, target):
319
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
320
target_len = len(target)
323
for s_begin, s_end, t_len, new_text in knit_delta:
324
true_n = s_begin - s_pos
327
# knit deltas do not provide reliable info about whether the
328
# last line of a file matches, due to eol handling.
329
if source[s_pos + n -1] != target[t_pos + n -1]:
332
yield s_pos, t_pos, n
333
t_pos += t_len + true_n
335
n = target_len - t_pos
337
if source[s_pos + n -1] != target[t_pos + n -1]:
340
yield s_pos, t_pos, n
341
yield s_pos + (target_len - t_pos), target_len, 0
344
class AnnotatedKnitContent(KnitContent):
345
"""Annotated content."""
347
def __init__(self, lines):
348
KnitContent.__init__(self)
352
"""Return a list of (origin, text) for each content line."""
353
lines = self._lines[:]
354
if self._should_strip_eol:
355
origin, last_line = lines[-1]
356
lines[-1] = (origin, last_line.rstrip('\n'))
359
def apply_delta(self, delta, new_version_id):
360
"""Apply delta to this object to become new_version_id."""
363
for start, end, count, delta_lines in delta:
364
lines[offset+start:offset+end] = delta_lines
365
offset = offset + (start - end) + count
369
lines = [text for origin, text in self._lines]
370
except ValueError, e:
371
# most commonly (only?) caused by the internal form of the knit
372
# missing annotation information because of a bug - see thread
374
raise KnitCorrupt(self,
375
"line in annotated knit missing annotation information: %s"
377
if self._should_strip_eol:
378
lines[-1] = lines[-1].rstrip('\n')
382
return AnnotatedKnitContent(self._lines[:])
385
class PlainKnitContent(KnitContent):
386
"""Unannotated content.
388
When annotate[_iter] is called on this content, the same version is reported
389
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
393
def __init__(self, lines, version_id):
394
KnitContent.__init__(self)
396
self._version_id = version_id
399
"""Return a list of (origin, text) for each content line."""
400
return [(self._version_id, line) for line in self._lines]
402
def apply_delta(self, delta, new_version_id):
403
"""Apply delta to this object to become new_version_id."""
406
for start, end, count, delta_lines in delta:
407
lines[offset+start:offset+end] = delta_lines
408
offset = offset + (start - end) + count
409
self._version_id = new_version_id
412
return PlainKnitContent(self._lines[:], self._version_id)
416
if self._should_strip_eol:
418
lines[-1] = lines[-1].rstrip('\n')
422
class _KnitFactory(object):
423
"""Base class for common Factory functions."""
425
def parse_record(self, version_id, record, record_details,
426
base_content, copy_base_content=True):
427
"""Parse a record into a full content object.
429
:param version_id: The official version id for this content
430
:param record: The data returned by read_records_iter()
431
:param record_details: Details about the record returned by
433
:param base_content: If get_build_details returns a compression_parent,
434
you must return a base_content here, else use None
435
:param copy_base_content: When building from the base_content, decide
436
you can either copy it and return a new object, or modify it in
438
:return: (content, delta) A Content object and possibly a line-delta,
441
method, noeol = record_details
442
if method == 'line-delta':
443
if copy_base_content:
444
content = base_content.copy()
446
content = base_content
447
delta = self.parse_line_delta(record, version_id)
448
content.apply_delta(delta, version_id)
450
content = self.parse_fulltext(record, version_id)
452
content._should_strip_eol = noeol
453
return (content, delta)
456
class KnitAnnotateFactory(_KnitFactory):
457
"""Factory for creating annotated Content objects."""
461
def make(self, lines, version_id):
462
num_lines = len(lines)
463
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
465
def parse_fulltext(self, content, version_id):
466
"""Convert fulltext to internal representation
468
fulltext content is of the format
469
revid(utf8) plaintext\n
470
internal representation is of the format:
473
# TODO: jam 20070209 The tests expect this to be returned as tuples,
474
# but the code itself doesn't really depend on that.
475
# Figure out a way to not require the overhead of turning the
476
# list back into tuples.
477
lines = [tuple(line.split(' ', 1)) for line in content]
478
return AnnotatedKnitContent(lines)
480
def parse_line_delta_iter(self, lines):
481
return iter(self.parse_line_delta(lines))
483
def parse_line_delta(self, lines, version_id, plain=False):
484
"""Convert a line based delta into internal representation.
486
line delta is in the form of:
487
intstart intend intcount
489
revid(utf8) newline\n
490
internal representation is
491
(start, end, count, [1..count tuples (revid, newline)])
493
:param plain: If True, the lines are returned as a plain
494
list without annotations, not as a list of (origin, content) tuples, i.e.
495
(start, end, count, [1..count newline])
502
def cache_and_return(line):
503
origin, text = line.split(' ', 1)
504
return cache.setdefault(origin, origin), text
506
# walk through the lines parsing.
507
# Note that the plain test is explicitly pulled out of the
508
# loop to minimise any performance impact
511
start, end, count = [int(n) for n in header.split(',')]
512
contents = [next().split(' ', 1)[1] for i in xrange(count)]
513
result.append((start, end, count, contents))
516
start, end, count = [int(n) for n in header.split(',')]
517
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
518
result.append((start, end, count, contents))
521
def get_fulltext_content(self, lines):
522
"""Extract just the content lines from a fulltext."""
523
return (line.split(' ', 1)[1] for line in lines)
525
def get_linedelta_content(self, lines):
526
"""Extract just the content from a line delta.
528
This doesn't return all of the extra information stored in a delta.
529
Only the actual content lines.
534
header = header.split(',')
535
count = int(header[2])
536
for i in xrange(count):
537
origin, text = next().split(' ', 1)
540
def lower_fulltext(self, content):
541
"""convert a fulltext content record into a serializable form.
543
see parse_fulltext which this inverts.
545
# TODO: jam 20070209 We only do the caching thing to make sure that
546
# the origin is a valid utf-8 line, eventually we could remove it
547
return ['%s %s' % (o, t) for o, t in content._lines]
549
def lower_line_delta(self, delta):
550
"""convert a delta into a serializable form.
552
See parse_line_delta which this inverts.
554
# TODO: jam 20070209 We only do the caching thing to make sure that
555
# the origin is a valid utf-8 line, eventually we could remove it
557
for start, end, c, lines in delta:
558
out.append('%d,%d,%d\n' % (start, end, c))
559
out.extend(origin + ' ' + text
560
for origin, text in lines)
563
def annotate(self, knit, key):
564
content = knit._get_content(key)
565
# adjust for the fact that serialised annotations are only key suffixes
567
if type(key) == tuple:
569
origins = content.annotate()
571
for origin, line in origins:
572
result.append((prefix + (origin,), line))
575
# XXX: This smells a bit. Why would key ever be a non-tuple here?
576
# Aren't keys defined to be tuples? -- spiv 20080618
577
return content.annotate()
580
class KnitPlainFactory(_KnitFactory):
581
"""Factory for creating plain Content objects."""
585
def make(self, lines, version_id):
586
return PlainKnitContent(lines, version_id)
588
def parse_fulltext(self, content, version_id):
589
"""This parses an unannotated fulltext.
591
Note that this is not a noop - the internal representation
592
has (versionid, line) - its just a constant versionid.
594
return self.make(content, version_id)
596
def parse_line_delta_iter(self, lines, version_id):
598
num_lines = len(lines)
599
while cur < num_lines:
602
start, end, c = [int(n) for n in header.split(',')]
603
yield start, end, c, lines[cur:cur+c]
606
def parse_line_delta(self, lines, version_id):
607
return list(self.parse_line_delta_iter(lines, version_id))
609
def get_fulltext_content(self, lines):
610
"""Extract just the content lines from a fulltext."""
613
def get_linedelta_content(self, lines):
614
"""Extract just the content from a line delta.
616
This doesn't return all of the extra information stored in a delta.
617
Only the actual content lines.
622
header = header.split(',')
623
count = int(header[2])
624
for i in xrange(count):
627
def lower_fulltext(self, content):
628
return content.text()
630
def lower_line_delta(self, delta):
632
for start, end, c, lines in delta:
633
out.append('%d,%d,%d\n' % (start, end, c))
637
def annotate(self, knit, key):
638
annotator = _KnitAnnotator(knit)
639
return annotator.annotate(key)
643
def make_file_factory(annotated, mapper):
644
"""Create a factory for creating a file based KnitVersionedFiles.
646
This is only functional enough to run interface tests, it doesn't try to
647
provide a full pack environment.
649
:param annotated: knit annotations are wanted.
650
:param mapper: The mapper from keys to paths.
652
def factory(transport):
653
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
654
access = _KnitKeyAccess(transport, mapper)
655
return KnitVersionedFiles(index, access, annotated=annotated)
659
def make_pack_factory(graph, delta, keylength):
660
"""Create a factory for creating a pack based VersionedFiles.
662
This is only functional enough to run interface tests, it doesn't try to
663
provide a full pack environment.
665
:param graph: Store a graph.
666
:param delta: Delta compress contents.
667
:param keylength: How long should keys be.
669
def factory(transport):
670
parents = graph or delta
676
max_delta_chain = 200
679
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
680
key_elements=keylength)
681
stream = transport.open_write_stream('newpack')
682
writer = pack.ContainerWriter(stream.write)
684
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
685
deltas=delta, add_callback=graph_index.add_nodes)
686
access = _DirectPackAccess({})
687
access.set_writer(writer, graph_index, (transport, 'newpack'))
688
result = KnitVersionedFiles(index, access,
689
max_delta_chain=max_delta_chain)
690
result.stream = stream
691
result.writer = writer
696
def cleanup_pack_knit(versioned_files):
697
versioned_files.stream.close()
698
versioned_files.writer.end()
701
class KnitVersionedFiles(VersionedFiles):
702
"""Storage for many versioned files using knit compression.
704
Backend storage is managed by indices and data objects.
706
:ivar _index: A _KnitGraphIndex or similar that can describe the
707
parents, graph, compression and data location of entries in this
708
KnitVersionedFiles. Note that this is only the index for
709
*this* vfs; if there are fallbacks they must be queried separately.
712
def __init__(self, index, data_access, max_delta_chain=200,
713
annotated=False, reload_func=None):
714
"""Create a KnitVersionedFiles with index and data_access.
716
:param index: The index for the knit data.
717
:param data_access: The access object to store and retrieve knit
719
:param max_delta_chain: The maximum number of deltas to permit during
720
insertion. Set to 0 to prohibit the use of deltas.
721
:param annotated: Set to True to cause annotations to be calculated and
722
stored during insertion.
723
:param reload_func: An function that can be called if we think we need
724
to reload the pack listing and try again. See
725
'bzrlib.repofmt.pack_repo.AggregateIndex' for the signature.
728
self._access = data_access
729
self._max_delta_chain = max_delta_chain
731
self._factory = KnitAnnotateFactory()
733
self._factory = KnitPlainFactory()
734
self._fallback_vfs = []
735
self._reload_func = reload_func
738
return "%s(%r, %r)" % (
739
self.__class__.__name__,
743
def add_fallback_versioned_files(self, a_versioned_files):
744
"""Add a source of texts for texts not present in this knit.
746
:param a_versioned_files: A VersionedFiles object.
748
self._fallback_vfs.append(a_versioned_files)
750
def add_lines(self, key, parents, lines, parent_texts=None,
751
left_matching_blocks=None, nostore_sha=None, random_id=False,
753
"""See VersionedFiles.add_lines()."""
754
self._index._check_write_ok()
755
self._check_add(key, lines, random_id, check_content)
757
# The caller might pass None if there is no graph data, but kndx
758
# indexes can't directly store that, so we give them
759
# an empty tuple instead.
761
return self._add(key, lines, parents,
762
parent_texts, left_matching_blocks, nostore_sha, random_id)
764
def _add(self, key, lines, parents, parent_texts,
765
left_matching_blocks, nostore_sha, random_id):
766
"""Add a set of lines on top of version specified by parents.
768
Any versions not present will be converted into ghosts.
770
# first thing, if the content is something we don't need to store, find
772
line_bytes = ''.join(lines)
773
digest = sha_string(line_bytes)
774
if nostore_sha == digest:
775
raise errors.ExistingContent
778
if parent_texts is None:
780
# Do a single query to ascertain parent presence; we only compress
781
# against parents in the same kvf.
782
present_parent_map = self._index.get_parent_map(parents)
783
for parent in parents:
784
if parent in present_parent_map:
785
present_parents.append(parent)
787
# Currently we can only compress against the left most present parent.
788
if (len(present_parents) == 0 or
789
present_parents[0] != parents[0]):
792
# To speed the extract of texts the delta chain is limited
793
# to a fixed number of deltas. This should minimize both
794
# I/O and the time spend applying deltas.
795
delta = self._check_should_delta(present_parents[0])
797
text_length = len(line_bytes)
800
if lines[-1][-1] != '\n':
801
# copy the contents of lines.
803
options.append('no-eol')
804
lines[-1] = lines[-1] + '\n'
808
if type(element) != str:
809
raise TypeError("key contains non-strings: %r" % (key,))
810
# Knit hunks are still last-element only
812
content = self._factory.make(lines, version_id)
813
if 'no-eol' in options:
814
# Hint to the content object that its text() call should strip the
816
content._should_strip_eol = True
817
if delta or (self._factory.annotated and len(present_parents) > 0):
818
# Merge annotations from parent texts if needed.
819
delta_hunks = self._merge_annotations(content, present_parents,
820
parent_texts, delta, self._factory.annotated,
821
left_matching_blocks)
824
options.append('line-delta')
825
store_lines = self._factory.lower_line_delta(delta_hunks)
826
size, bytes = self._record_to_data(key, digest,
829
options.append('fulltext')
830
# isinstance is slower and we have no hierarchy.
831
if self._factory.__class__ == KnitPlainFactory:
832
# Use the already joined bytes saving iteration time in
834
size, bytes = self._record_to_data(key, digest,
837
# get mixed annotation + content and feed it into the
839
store_lines = self._factory.lower_fulltext(content)
840
size, bytes = self._record_to_data(key, digest,
843
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
844
self._index.add_records(
845
((key, options, access_memo, parents),),
847
return digest, text_length, content
849
def annotate(self, key):
850
"""See VersionedFiles.annotate."""
851
return self._factory.annotate(self, key)
853
def check(self, progress_bar=None):
854
"""See VersionedFiles.check()."""
855
# This doesn't actually test extraction of everything, but that will
856
# impact 'bzr check' substantially, and needs to be integrated with
857
# care. However, it does check for the obvious problem of a delta with
859
keys = self._index.keys()
860
parent_map = self.get_parent_map(keys)
862
if self._index.get_method(key) != 'fulltext':
863
compression_parent = parent_map[key][0]
864
if compression_parent not in parent_map:
865
raise errors.KnitCorrupt(self,
866
"Missing basis parent %s for %s" % (
867
compression_parent, key))
868
for fallback_vfs in self._fallback_vfs:
871
def _check_add(self, key, lines, random_id, check_content):
872
"""check that version_id and lines are safe to add."""
874
if contains_whitespace(version_id):
875
raise InvalidRevisionId(version_id, self)
876
self.check_not_reserved_id(version_id)
877
# TODO: If random_id==False and the key is already present, we should
878
# probably check that the existing content is identical to what is
879
# being inserted, and otherwise raise an exception. This would make
880
# the bundle code simpler.
882
self._check_lines_not_unicode(lines)
883
self._check_lines_are_lines(lines)
885
def _check_header(self, key, line):
886
rec = self._split_header(line)
887
self._check_header_version(rec, key[-1])
890
def _check_header_version(self, rec, version_id):
891
"""Checks the header version on original format knit records.
893
These have the last component of the key embedded in the record.
895
if rec[1] != version_id:
896
raise KnitCorrupt(self,
897
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
899
def _check_should_delta(self, parent):
900
"""Iterate back through the parent listing, looking for a fulltext.
902
This is used when we want to decide whether to add a delta or a new
903
fulltext. It searches for _max_delta_chain parents. When it finds a
904
fulltext parent, it sees if the total size of the deltas leading up to
905
it is large enough to indicate that we want a new full text anyway.
907
Return True if we should create a new delta, False if we should use a
912
for count in xrange(self._max_delta_chain):
913
# XXX: Collapse these two queries:
915
# Note that this only looks in the index of this particular
916
# KnitVersionedFiles, not in the fallbacks. This ensures that
917
# we won't store a delta spanning physical repository
919
method = self._index.get_method(parent)
920
except RevisionNotPresent:
921
# Some basis is not locally present: always delta
923
index, pos, size = self._index.get_position(parent)
924
if method == 'fulltext':
928
# We don't explicitly check for presence because this is in an
929
# inner loop, and if it's missing it'll fail anyhow.
930
# TODO: This should be asking for compression parent, not graph
932
parent = self._index.get_parent_map([parent])[parent][0]
934
# We couldn't find a fulltext, so we must create a new one
936
# Simple heuristic - if the total I/O wold be greater as a delta than
937
# the originally installed fulltext, we create a new fulltext.
938
return fulltext_size > delta_size
940
def _build_details_to_components(self, build_details):
941
"""Convert a build_details tuple to a position tuple."""
942
# record_details, access_memo, compression_parent
943
return build_details[3], build_details[0], build_details[1]
945
def _get_components_positions(self, keys, allow_missing=False):
946
"""Produce a map of position data for the components of keys.
948
This data is intended to be used for retrieving the knit records.
950
A dict of key to (record_details, index_memo, next, parents) is
952
method is the way referenced data should be applied.
953
index_memo is the handle to pass to the data access to actually get the
955
next is the build-parent of the version, or None for fulltexts.
956
parents is the version_ids of the parents of this version
958
:param allow_missing: If True do not raise an error on a missing component,
962
pending_components = keys
963
while pending_components:
964
build_details = self._index.get_build_details(pending_components)
965
current_components = set(pending_components)
966
pending_components = set()
967
for key, details in build_details.iteritems():
968
(index_memo, compression_parent, parents,
969
record_details) = details
970
method = record_details[0]
971
if compression_parent is not None:
972
pending_components.add(compression_parent)
973
component_data[key] = self._build_details_to_components(details)
974
missing = current_components.difference(build_details)
975
if missing and not allow_missing:
976
raise errors.RevisionNotPresent(missing.pop(), self)
977
return component_data
979
def _get_content(self, key, parent_texts={}):
980
"""Returns a content object that makes up the specified
982
cached_version = parent_texts.get(key, None)
983
if cached_version is not None:
984
# Ensure the cache dict is valid.
985
if not self.get_parent_map([key]):
986
raise RevisionNotPresent(key, self)
987
return cached_version
988
text_map, contents_map = self._get_content_maps([key])
989
return contents_map[key]
991
def _get_content_maps(self, keys, nonlocal_keys=None):
992
"""Produce maps of text and KnitContents
994
:param keys: The keys to produce content maps for.
995
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
996
which are known to not be in this knit, but rather in one of the
998
:return: (text_map, content_map) where text_map contains the texts for
999
the requested versions and content_map contains the KnitContents.
1001
# FUTURE: This function could be improved for the 'extract many' case
1002
# by tracking each component and only doing the copy when the number of
1003
# children than need to apply delta's to it is > 1 or it is part of the
1006
multiple_versions = len(keys) != 1
1007
record_map = self._get_record_map(keys, allow_missing=True)
1012
if nonlocal_keys is None:
1013
nonlocal_keys = set()
1015
nonlocal_keys = frozenset(nonlocal_keys)
1016
missing_keys = set(nonlocal_keys)
1017
for source in self._fallback_vfs:
1018
if not missing_keys:
1020
for record in source.get_record_stream(missing_keys,
1022
if record.storage_kind == 'absent':
1024
missing_keys.remove(record.key)
1025
lines = split_lines(record.get_bytes_as('fulltext'))
1026
text_map[record.key] = lines
1027
content_map[record.key] = PlainKnitContent(lines, record.key)
1028
if record.key in keys:
1029
final_content[record.key] = content_map[record.key]
1031
if key in nonlocal_keys:
1036
while cursor is not None:
1038
record, record_details, digest, next = record_map[cursor]
1040
raise RevisionNotPresent(cursor, self)
1041
components.append((cursor, record, record_details, digest))
1043
if cursor in content_map:
1044
# no need to plan further back
1045
components.append((cursor, None, None, None))
1049
for (component_id, record, record_details,
1050
digest) in reversed(components):
1051
if component_id in content_map:
1052
content = content_map[component_id]
1054
content, delta = self._factory.parse_record(key[-1],
1055
record, record_details, content,
1056
copy_base_content=multiple_versions)
1057
if multiple_versions:
1058
content_map[component_id] = content
1060
final_content[key] = content
1062
# digest here is the digest from the last applied component.
1063
text = content.text()
1064
actual_sha = sha_strings(text)
1065
if actual_sha != digest:
1066
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
1067
text_map[key] = text
1068
return text_map, final_content
1070
def get_parent_map(self, keys):
1071
"""Get a map of the graph parents of keys.
1073
:param keys: The keys to look up parents for.
1074
:return: A mapping from keys to parents. Absent keys are absent from
1077
return self._get_parent_map_with_sources(keys)[0]
1079
def _get_parent_map_with_sources(self, keys):
1080
"""Get a map of the parents of keys.
1082
:param keys: The keys to look up parents for.
1083
:return: A tuple. The first element is a mapping from keys to parents.
1084
Absent keys are absent from the mapping. The second element is a
1085
list with the locations each key was found in. The first element
1086
is the in-this-knit parents, the second the first fallback source,
1090
sources = [self._index] + self._fallback_vfs
1093
for source in sources:
1096
new_result = source.get_parent_map(missing)
1097
source_results.append(new_result)
1098
result.update(new_result)
1099
missing.difference_update(set(new_result))
1100
return result, source_results
1102
def _get_record_map(self, keys, allow_missing=False):
1103
"""Produce a dictionary of knit records.
1105
:return: {key:(record, record_details, digest, next)}
1107
data returned from read_records
1109
opaque information to pass to parse_record
1111
SHA1 digest of the full text after all steps are done
1113
build-parent of the version, i.e. the leftmost ancestor.
1114
Will be None if the record is not a delta.
1115
:param keys: The keys to build a map for
1116
:param allow_missing: If some records are missing, rather than
1117
error, just return the data that could be generated.
1119
# This retries the whole request if anything fails. Potentially we
1120
# could be a bit more selective. We could track the keys whose records
1121
# we have successfully found, and then only request the new records
1122
# from there. However, _get_components_positions grabs the whole build
1123
# chain, which means we'll likely try to grab the same records again
1124
# anyway. Also, can the build chains change as part of a pack
1125
# operation? We wouldn't want to end up with a broken chain.
1128
position_map = self._get_components_positions(keys,
1129
allow_missing=allow_missing)
1130
# key = component_id, r = record_details, i_m = index_memo,
1132
records = [(key, i_m) for key, (r, i_m, n)
1133
in position_map.iteritems()]
1135
for key, record, digest in self._read_records_iter(records):
1136
(record_details, index_memo, next) = position_map[key]
1137
record_map[key] = record, record_details, digest, next
1139
except errors.RetryWithNewPacks, e:
1140
self._access.reload_or_raise(e)
1142
def _split_by_prefix(self, keys):
1143
"""For the given keys, split them up based on their prefix.
1145
To keep memory pressure somewhat under control, split the
1146
requests back into per-file-id requests, otherwise "bzr co"
1147
extracts the full tree into memory before writing it to disk.
1148
This should be revisited if _get_content_maps() can ever cross
1151
:param keys: An iterable of key tuples
1152
:return: A dict of {prefix: [key_list]}
1154
split_by_prefix = {}
1157
split_by_prefix.setdefault('', []).append(key)
1159
split_by_prefix.setdefault(key[0], []).append(key)
1160
return split_by_prefix
1162
def get_record_stream(self, keys, ordering, include_delta_closure):
1163
"""Get a stream of records for keys.
1165
:param keys: The keys to include.
1166
:param ordering: Either 'unordered' or 'topological'. A topologically
1167
sorted stream has compression parents strictly before their
1169
:param include_delta_closure: If True then the closure across any
1170
compression parents will be included (in the opaque data).
1171
:return: An iterator of ContentFactory objects, each of which is only
1172
valid until the iterator is advanced.
1174
# keys might be a generator
1178
if not self._index.has_graph:
1179
# Cannot topological order when no graph has been stored.
1180
ordering = 'unordered'
1182
remaining_keys = keys
1185
keys = set(remaining_keys)
1186
for content_factory in self._get_remaining_record_stream(keys,
1187
ordering, include_delta_closure):
1188
remaining_keys.discard(content_factory.key)
1189
yield content_factory
1191
except errors.RetryWithNewPacks, e:
1192
self._access.reload_or_raise(e)
1194
def _get_remaining_record_stream(self, keys, ordering,
1195
include_delta_closure):
1196
"""This function is the 'retry' portion for get_record_stream."""
1197
if include_delta_closure:
1198
positions = self._get_components_positions(keys, allow_missing=True)
1200
build_details = self._index.get_build_details(keys)
1202
# (record_details, access_memo, compression_parent_key)
1203
positions = dict((key, self._build_details_to_components(details))
1204
for key, details in build_details.iteritems())
1205
absent_keys = keys.difference(set(positions))
1206
# There may be more absent keys : if we're missing the basis component
1207
# and are trying to include the delta closure.
1208
if include_delta_closure:
1209
needed_from_fallback = set()
1210
# Build up reconstructable_keys dict. key:True in this dict means
1211
# the key can be reconstructed.
1212
reconstructable_keys = {}
1216
chain = [key, positions[key][2]]
1218
needed_from_fallback.add(key)
1221
while chain[-1] is not None:
1222
if chain[-1] in reconstructable_keys:
1223
result = reconstructable_keys[chain[-1]]
1227
chain.append(positions[chain[-1]][2])
1229
# missing basis component
1230
needed_from_fallback.add(chain[-1])
1233
for chain_key in chain[:-1]:
1234
reconstructable_keys[chain_key] = result
1236
needed_from_fallback.add(key)
1237
# Double index lookups here : need a unified api ?
1238
global_map, parent_maps = self._get_parent_map_with_sources(keys)
1239
if ordering == 'topological':
1240
# Global topological sort
1241
present_keys = tsort.topo_sort(global_map)
1242
# Now group by source:
1244
current_source = None
1245
for key in present_keys:
1246
for parent_map in parent_maps:
1247
if key in parent_map:
1248
key_source = parent_map
1250
if current_source is not key_source:
1251
source_keys.append((key_source, []))
1252
current_source = key_source
1253
source_keys[-1][1].append(key)
1255
if ordering != 'unordered':
1256
raise AssertionError('valid values for ordering are:'
1257
' "unordered" or "topological" not: %r'
1259
# Just group by source; remote sources first.
1262
for parent_map in reversed(parent_maps):
1263
source_keys.append((parent_map, []))
1264
for key in parent_map:
1265
present_keys.append(key)
1266
source_keys[-1][1].append(key)
1267
# We have been requested to return these records in an order that
1268
# suits us. So we ask the index to give us an optimally sorted
1270
for source, sub_keys in source_keys:
1271
if source is parent_maps[0]:
1272
# Only sort the keys for this VF
1273
self._index._sort_keys_by_io(sub_keys, positions)
1274
absent_keys = keys - set(global_map)
1275
for key in absent_keys:
1276
yield AbsentContentFactory(key)
1277
# restrict our view to the keys we can answer.
1278
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1279
# XXX: At that point we need to consider the impact of double reads by
1280
# utilising components multiple times.
1281
if include_delta_closure:
1282
# XXX: get_content_maps performs its own index queries; allow state
1284
non_local_keys = needed_from_fallback - absent_keys
1285
prefix_split_keys = self._split_by_prefix(present_keys)
1286
prefix_split_non_local_keys = self._split_by_prefix(non_local_keys)
1287
for prefix, keys in prefix_split_keys.iteritems():
1288
non_local = prefix_split_non_local_keys.get(prefix, [])
1289
non_local = set(non_local)
1290
text_map, _ = self._get_content_maps(keys, non_local)
1292
lines = text_map.pop(key)
1293
yield ChunkedContentFactory(key, global_map[key], None,
1296
for source, keys in source_keys:
1297
if source is parent_maps[0]:
1298
# this KnitVersionedFiles
1299
records = [(key, positions[key][1]) for key in keys]
1300
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1301
(record_details, index_memo, _) = positions[key]
1302
yield KnitContentFactory(key, global_map[key],
1303
record_details, sha1, raw_data, self._factory.annotated, None)
1305
vf = self._fallback_vfs[parent_maps.index(source) - 1]
1306
for record in vf.get_record_stream(keys, ordering,
1307
include_delta_closure):
1310
def get_sha1s(self, keys):
1311
"""See VersionedFiles.get_sha1s()."""
1313
record_map = self._get_record_map(missing, allow_missing=True)
1315
for key, details in record_map.iteritems():
1316
if key not in missing:
1318
# record entry 2 is the 'digest'.
1319
result[key] = details[2]
1320
missing.difference_update(set(result))
1321
for source in self._fallback_vfs:
1324
new_result = source.get_sha1s(missing)
1325
result.update(new_result)
1326
missing.difference_update(set(new_result))
1329
def insert_record_stream(self, stream):
1330
"""Insert a record stream into this container.
1332
:param stream: A stream of records to insert.
1334
:seealso VersionedFiles.get_record_stream:
1336
def get_adapter(adapter_key):
1338
return adapters[adapter_key]
1340
adapter_factory = adapter_registry.get(adapter_key)
1341
adapter = adapter_factory(self)
1342
adapters[adapter_key] = adapter
1345
if self._factory.annotated:
1346
# self is annotated, we need annotated knits to use directly.
1347
annotated = "annotated-"
1350
# self is not annotated, but we can strip annotations cheaply.
1352
convertibles = set(["knit-annotated-ft-gz"])
1353
if self._max_delta_chain:
1354
delta_types.add("knit-annotated-delta-gz")
1355
convertibles.add("knit-annotated-delta-gz")
1356
# The set of types we can cheaply adapt without needing basis texts.
1357
native_types = set()
1358
if self._max_delta_chain:
1359
native_types.add("knit-%sdelta-gz" % annotated)
1360
delta_types.add("knit-%sdelta-gz" % annotated)
1361
native_types.add("knit-%sft-gz" % annotated)
1362
knit_types = native_types.union(convertibles)
1364
# Buffer all index entries that we can't add immediately because their
1365
# basis parent is missing. We don't buffer all because generating
1366
# annotations may require access to some of the new records. However we
1367
# can't generate annotations from new deltas until their basis parent
1368
# is present anyway, so we get away with not needing an index that
1369
# includes the new keys.
1371
# See <http://launchpad.net/bugs/300177> about ordering of compression
1372
# parents in the records - to be conservative, we insist that all
1373
# parents must be present to avoid expanding to a fulltext.
1375
# key = basis_parent, value = index entry to add
1376
buffered_index_entries = {}
1377
for record in stream:
1378
parents = record.parents
1379
if record.storage_kind in delta_types:
1380
# TODO: eventually the record itself should track
1381
# compression_parent
1382
compression_parent = parents[0]
1384
compression_parent = None
1385
# Raise an error when a record is missing.
1386
if record.storage_kind == 'absent':
1387
raise RevisionNotPresent([record.key], self)
1388
elif ((record.storage_kind in knit_types)
1389
and (compression_parent is None
1390
or not self._fallback_vfs
1391
or self._index.has_key(compression_parent)
1392
or not self.has_key(compression_parent))):
1393
# we can insert the knit record literally if either it has no
1394
# compression parent OR we already have its basis in this kvf
1395
# OR the basis is not present even in the fallbacks. In the
1396
# last case it will either turn up later in the stream and all
1397
# will be well, or it won't turn up at all and we'll raise an
1400
# TODO: self.has_key is somewhat redundant with
1401
# self._index.has_key; we really want something that directly
1402
# asks if it's only present in the fallbacks. -- mbp 20081119
1403
if record.storage_kind not in native_types:
1405
adapter_key = (record.storage_kind, "knit-delta-gz")
1406
adapter = get_adapter(adapter_key)
1408
adapter_key = (record.storage_kind, "knit-ft-gz")
1409
adapter = get_adapter(adapter_key)
1410
bytes = adapter.get_bytes(
1411
record, record.get_bytes_as(record.storage_kind))
1413
bytes = record.get_bytes_as(record.storage_kind)
1414
options = [record._build_details[0]]
1415
if record._build_details[1]:
1416
options.append('no-eol')
1417
# Just blat it across.
1418
# Note: This does end up adding data on duplicate keys. As
1419
# modern repositories use atomic insertions this should not
1420
# lead to excessive growth in the event of interrupted fetches.
1421
# 'knit' repositories may suffer excessive growth, but as a
1422
# deprecated format this is tolerable. It can be fixed if
1423
# needed by in the kndx index support raising on a duplicate
1424
# add with identical parents and options.
1425
access_memo = self._access.add_raw_records(
1426
[(record.key, len(bytes))], bytes)[0]
1427
index_entry = (record.key, options, access_memo, parents)
1429
if 'fulltext' not in options:
1430
# Not a fulltext, so we need to make sure the compression
1431
# parent will also be present.
1432
# Note that pack backed knits don't need to buffer here
1433
# because they buffer all writes to the transaction level,
1434
# but we don't expose that difference at the index level. If
1435
# the query here has sufficient cost to show up in
1436
# profiling we should do that.
1438
# They're required to be physically in this
1439
# KnitVersionedFiles, not in a fallback.
1440
if not self._index.has_key(compression_parent):
1441
pending = buffered_index_entries.setdefault(
1442
compression_parent, [])
1443
pending.append(index_entry)
1446
self._index.add_records([index_entry])
1447
elif (record.storage_kind == 'fulltext'
1448
or record.storage_kind == 'chunked'):
1449
self.add_lines(record.key, parents,
1450
split_lines(record.get_bytes_as('fulltext')))
1452
# Not a fulltext, and not suitable for direct insertion as a
1453
# delta, either because it's not the right format, or this
1454
# KnitVersionedFiles doesn't permit deltas (_max_delta_chain ==
1455
# 0) or because it depends on a base only present in the
1457
adapter_key = record.storage_kind, 'fulltext'
1458
adapter = get_adapter(adapter_key)
1459
lines = split_lines(adapter.get_bytes(
1460
record, record.get_bytes_as(record.storage_kind)))
1462
self.add_lines(record.key, parents, lines)
1463
except errors.RevisionAlreadyPresent:
1465
# Add any records whose basis parent is now available.
1466
added_keys = [record.key]
1468
key = added_keys.pop(0)
1469
if key in buffered_index_entries:
1470
index_entries = buffered_index_entries[key]
1471
self._index.add_records(index_entries)
1473
[index_entry[0] for index_entry in index_entries])
1474
del buffered_index_entries[key]
1475
# If there were any deltas which had a missing basis parent, error.
1476
if buffered_index_entries:
1477
from pprint import pformat
1478
raise errors.BzrCheckError(
1479
"record_stream refers to compression parents not in %r:\n%s"
1480
% (self, pformat(sorted(buffered_index_entries.keys()))))
1482
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1483
"""Iterate over the lines in the versioned files from keys.
1485
This may return lines from other keys. Each item the returned
1486
iterator yields is a tuple of a line and a text version that that line
1487
is present in (not introduced in).
1489
Ordering of results is in whatever order is most suitable for the
1490
underlying storage format.
1492
If a progress bar is supplied, it may be used to indicate progress.
1493
The caller is responsible for cleaning up progress bars (because this
1497
* Lines are normalised by the underlying store: they will all have \\n
1499
* Lines are returned in arbitrary order.
1500
* If a requested key did not change any lines (or didn't have any
1501
lines), it may not be mentioned at all in the result.
1503
:return: An iterator over (line, key).
1506
pb = progress.DummyProgress()
1512
# we don't care about inclusions, the caller cares.
1513
# but we need to setup a list of records to visit.
1514
# we need key, position, length
1516
build_details = self._index.get_build_details(keys)
1517
for key, details in build_details.iteritems():
1519
key_records.append((key, details[0]))
1520
records_iter = enumerate(self._read_records_iter(key_records))
1521
for (key_idx, (key, data, sha_value)) in records_iter:
1522
pb.update('Walking content.', key_idx, total)
1523
compression_parent = build_details[key][1]
1524
if compression_parent is None:
1526
line_iterator = self._factory.get_fulltext_content(data)
1529
line_iterator = self._factory.get_linedelta_content(data)
1530
# Now that we are yielding the data for this key, remove it
1533
# XXX: It might be more efficient to yield (key,
1534
# line_iterator) in the future. However for now, this is a
1535
# simpler change to integrate into the rest of the
1536
# codebase. RBC 20071110
1537
for line in line_iterator:
1540
except errors.RetryWithNewPacks, e:
1541
self._access.reload_or_raise(e)
1542
# If there are still keys we've not yet found, we look in the fallback
1543
# vfs, and hope to find them there. Note that if the keys are found
1544
# but had no changes or no content, the fallback may not return
1546
if keys and not self._fallback_vfs:
1547
# XXX: strictly the second parameter is meant to be the file id
1548
# but it's not easily accessible here.
1549
raise RevisionNotPresent(keys, repr(self))
1550
for source in self._fallback_vfs:
1554
for line, key in source.iter_lines_added_or_present_in_keys(keys):
1555
source_keys.add(key)
1557
keys.difference_update(source_keys)
1558
pb.update('Walking content.', total, total)
1560
def _make_line_delta(self, delta_seq, new_content):
1561
"""Generate a line delta from delta_seq and new_content."""
1563
for op in delta_seq.get_opcodes():
1564
if op[0] == 'equal':
1566
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1569
def _merge_annotations(self, content, parents, parent_texts={},
1570
delta=None, annotated=None,
1571
left_matching_blocks=None):
1572
"""Merge annotations for content and generate deltas.
1574
This is done by comparing the annotations based on changes to the text
1575
and generating a delta on the resulting full texts. If annotations are
1576
not being created then a simple delta is created.
1578
if left_matching_blocks is not None:
1579
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1583
for parent_key in parents:
1584
merge_content = self._get_content(parent_key, parent_texts)
1585
if (parent_key == parents[0] and delta_seq is not None):
1588
seq = patiencediff.PatienceSequenceMatcher(
1589
None, merge_content.text(), content.text())
1590
for i, j, n in seq.get_matching_blocks():
1593
# this copies (origin, text) pairs across to the new
1594
# content for any line that matches the last-checked
1596
content._lines[j:j+n] = merge_content._lines[i:i+n]
1597
# XXX: Robert says the following block is a workaround for a
1598
# now-fixed bug and it can probably be deleted. -- mbp 20080618
1599
if content._lines and content._lines[-1][1][-1] != '\n':
1600
# The copied annotation was from a line without a trailing EOL,
1601
# reinstate one for the content object, to ensure correct
1603
line = content._lines[-1][1] + '\n'
1604
content._lines[-1] = (content._lines[-1][0], line)
1606
if delta_seq is None:
1607
reference_content = self._get_content(parents[0], parent_texts)
1608
new_texts = content.text()
1609
old_texts = reference_content.text()
1610
delta_seq = patiencediff.PatienceSequenceMatcher(
1611
None, old_texts, new_texts)
1612
return self._make_line_delta(delta_seq, content)
1614
def _parse_record(self, version_id, data):
1615
"""Parse an original format knit record.
1617
These have the last element of the key only present in the stored data.
1619
rec, record_contents = self._parse_record_unchecked(data)
1620
self._check_header_version(rec, version_id)
1621
return record_contents, rec[3]
1623
def _parse_record_header(self, key, raw_data):
1624
"""Parse a record header for consistency.
1626
:return: the header and the decompressor stream.
1627
as (stream, header_record)
1629
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(raw_data))
1632
rec = self._check_header(key, df.readline())
1633
except Exception, e:
1634
raise KnitCorrupt(self,
1635
"While reading {%s} got %s(%s)"
1636
% (key, e.__class__.__name__, str(e)))
1639
def _parse_record_unchecked(self, data):
1641
# 4168 calls in 2880 217 internal
1642
# 4168 calls to _parse_record_header in 2121
1643
# 4168 calls to readlines in 330
1644
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(data))
1646
record_contents = df.readlines()
1647
except Exception, e:
1648
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1649
(data, e.__class__.__name__, str(e)))
1650
header = record_contents.pop(0)
1651
rec = self._split_header(header)
1652
last_line = record_contents.pop()
1653
if len(record_contents) != int(rec[2]):
1654
raise KnitCorrupt(self,
1655
'incorrect number of lines %s != %s'
1656
' for version {%s} %s'
1657
% (len(record_contents), int(rec[2]),
1658
rec[1], record_contents))
1659
if last_line != 'end %s\n' % rec[1]:
1660
raise KnitCorrupt(self,
1661
'unexpected version end line %r, wanted %r'
1662
% (last_line, rec[1]))
1664
return rec, record_contents
1666
def _read_records_iter(self, records):
1667
"""Read text records from data file and yield result.
1669
The result will be returned in whatever is the fastest to read.
1670
Not by the order requested. Also, multiple requests for the same
1671
record will only yield 1 response.
1672
:param records: A list of (key, access_memo) entries
1673
:return: Yields (key, contents, digest) in the order
1674
read, not the order requested
1679
# XXX: This smells wrong, IO may not be getting ordered right.
1680
needed_records = sorted(set(records), key=operator.itemgetter(1))
1681
if not needed_records:
1684
# The transport optimizes the fetching as well
1685
# (ie, reads continuous ranges.)
1686
raw_data = self._access.get_raw_records(
1687
[index_memo for key, index_memo in needed_records])
1689
for (key, index_memo), data in \
1690
izip(iter(needed_records), raw_data):
1691
content, digest = self._parse_record(key[-1], data)
1692
yield key, content, digest
1694
def _read_records_iter_raw(self, records):
1695
"""Read text records from data file and yield raw data.
1697
This unpacks enough of the text record to validate the id is
1698
as expected but thats all.
1700
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1702
# setup an iterator of the external records:
1703
# uses readv so nice and fast we hope.
1705
# grab the disk data needed.
1706
needed_offsets = [index_memo for key, index_memo
1708
raw_records = self._access.get_raw_records(needed_offsets)
1710
for key, index_memo in records:
1711
data = raw_records.next()
1712
# validate the header (note that we can only use the suffix in
1713
# current knit records).
1714
df, rec = self._parse_record_header(key, data)
1716
yield key, data, rec[3]
1718
def _record_to_data(self, key, digest, lines, dense_lines=None):
1719
"""Convert key, digest, lines into a raw data block.
1721
:param key: The key of the record. Currently keys are always serialised
1722
using just the trailing component.
1723
:param dense_lines: The bytes of lines but in a denser form. For
1724
instance, if lines is a list of 1000 bytestrings each ending in \n,
1725
dense_lines may be a list with one line in it, containing all the
1726
1000's lines and their \n's. Using dense_lines if it is already
1727
known is a win because the string join to create bytes in this
1728
function spends less time resizing the final string.
1729
:return: (len, a StringIO instance with the raw data ready to read.)
1731
# Note: using a string copy here increases memory pressure with e.g.
1732
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1733
# when doing the initial commit of a mozilla tree. RBC 20070921
1734
bytes = ''.join(chain(
1735
["version %s %d %s\n" % (key[-1],
1738
dense_lines or lines,
1739
["end %s\n" % key[-1]]))
1740
if type(bytes) != str:
1741
raise AssertionError(
1742
'data must be plain bytes was %s' % type(bytes))
1743
if lines and lines[-1][-1] != '\n':
1744
raise ValueError('corrupt lines value %r' % lines)
1745
compressed_bytes = tuned_gzip.bytes_to_gzip(bytes)
1746
return len(compressed_bytes), compressed_bytes
1748
def _split_header(self, line):
1751
raise KnitCorrupt(self,
1752
'unexpected number of elements in record header')
1756
"""See VersionedFiles.keys."""
1757
if 'evil' in debug.debug_flags:
1758
trace.mutter_callsite(2, "keys scales with size of history")
1759
sources = [self._index] + self._fallback_vfs
1761
for source in sources:
1762
result.update(source.keys())
1766
class _KndxIndex(object):
1767
"""Manages knit index files
1769
The index is kept in memory and read on startup, to enable
1770
fast lookups of revision information. The cursor of the index
1771
file is always pointing to the end, making it easy to append
1774
_cache is a cache for fast mapping from version id to a Index
1777
_history is a cache for fast mapping from indexes to version ids.
1779
The index data format is dictionary compressed when it comes to
1780
parent references; a index entry may only have parents that with a
1781
lover index number. As a result, the index is topological sorted.
1783
Duplicate entries may be written to the index for a single version id
1784
if this is done then the latter one completely replaces the former:
1785
this allows updates to correct version and parent information.
1786
Note that the two entries may share the delta, and that successive
1787
annotations and references MUST point to the first entry.
1789
The index file on disc contains a header, followed by one line per knit
1790
record. The same revision can be present in an index file more than once.
1791
The first occurrence gets assigned a sequence number starting from 0.
1793
The format of a single line is
1794
REVISION_ID FLAGS BYTE_OFFSET LENGTH( PARENT_ID|PARENT_SEQUENCE_ID)* :\n
1795
REVISION_ID is a utf8-encoded revision id
1796
FLAGS is a comma separated list of flags about the record. Values include
1797
no-eol, line-delta, fulltext.
1798
BYTE_OFFSET is the ascii representation of the byte offset in the data file
1799
that the the compressed data starts at.
1800
LENGTH is the ascii representation of the length of the data file.
1801
PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
1803
PARENT_SEQUENCE_ID the ascii representation of the sequence number of a
1804
revision id already in the knit that is a parent of REVISION_ID.
1805
The ' :' marker is the end of record marker.
1808
when a write is interrupted to the index file, it will result in a line
1809
that does not end in ' :'. If the ' :' is not present at the end of a line,
1810
or at the end of the file, then the record that is missing it will be
1811
ignored by the parser.
1813
When writing new records to the index file, the data is preceded by '\n'
1814
to ensure that records always start on new lines even if the last write was
1815
interrupted. As a result its normal for the last line in the index to be
1816
missing a trailing newline. One can be added with no harmful effects.
1818
:ivar _kndx_cache: dict from prefix to the old state of KnitIndex objects,
1819
where prefix is e.g. the (fileid,) for .texts instances or () for
1820
constant-mapped things like .revisions, and the old state is
1821
tuple(history_vector, cache_dict). This is used to prevent having an
1822
ABI change with the C extension that reads .kndx files.
1825
HEADER = "# bzr knit index 8\n"
1827
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1828
"""Create a _KndxIndex on transport using mapper."""
1829
self._transport = transport
1830
self._mapper = mapper
1831
self._get_scope = get_scope
1832
self._allow_writes = allow_writes
1833
self._is_locked = is_locked
1835
self.has_graph = True
1837
def add_records(self, records, random_id=False):
1838
"""Add multiple records to the index.
1840
:param records: a list of tuples:
1841
(key, options, access_memo, parents).
1842
:param random_id: If True the ids being added were randomly generated
1843
and no check for existence will be performed.
1846
for record in records:
1849
path = self._mapper.map(key) + '.kndx'
1850
path_keys = paths.setdefault(path, (prefix, []))
1851
path_keys[1].append(record)
1852
for path in sorted(paths):
1853
prefix, path_keys = paths[path]
1854
self._load_prefixes([prefix])
1856
orig_history = self._kndx_cache[prefix][1][:]
1857
orig_cache = self._kndx_cache[prefix][0].copy()
1860
for key, options, (_, pos, size), parents in path_keys:
1862
# kndx indices cannot be parentless.
1864
line = "\n%s %s %s %s %s :" % (
1865
key[-1], ','.join(options), pos, size,
1866
self._dictionary_compress(parents))
1867
if type(line) != str:
1868
raise AssertionError(
1869
'data must be utf8 was %s' % type(line))
1871
self._cache_key(key, options, pos, size, parents)
1872
if len(orig_history):
1873
self._transport.append_bytes(path, ''.join(lines))
1875
self._init_index(path, lines)
1877
# If any problems happen, restore the original values and re-raise
1878
self._kndx_cache[prefix] = (orig_cache, orig_history)
1881
def _cache_key(self, key, options, pos, size, parent_keys):
1882
"""Cache a version record in the history array and index cache.
1884
This is inlined into _load_data for performance. KEEP IN SYNC.
1885
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1889
version_id = key[-1]
1890
# last-element only for compatibilty with the C load_data.
1891
parents = tuple(parent[-1] for parent in parent_keys)
1892
for parent in parent_keys:
1893
if parent[:-1] != prefix:
1894
raise ValueError("mismatched prefixes for %r, %r" % (
1896
cache, history = self._kndx_cache[prefix]
1897
# only want the _history index to reference the 1st index entry
1899
if version_id not in cache:
1900
index = len(history)
1901
history.append(version_id)
1903
index = cache[version_id][5]
1904
cache[version_id] = (version_id,
1911
def check_header(self, fp):
1912
line = fp.readline()
1914
# An empty file can actually be treated as though the file doesn't
1916
raise errors.NoSuchFile(self)
1917
if line != self.HEADER:
1918
raise KnitHeaderError(badline=line, filename=self)
1920
def _check_read(self):
1921
if not self._is_locked():
1922
raise errors.ObjectNotLocked(self)
1923
if self._get_scope() != self._scope:
1926
def _check_write_ok(self):
1927
"""Assert if not writes are permitted."""
1928
if not self._is_locked():
1929
raise errors.ObjectNotLocked(self)
1930
if self._get_scope() != self._scope:
1932
if self._mode != 'w':
1933
raise errors.ReadOnlyObjectDirtiedError(self)
1935
def get_build_details(self, keys):
1936
"""Get the method, index_memo and compression parent for keys.
1938
Ghosts are omitted from the result.
1940
:param keys: An iterable of keys.
1941
:return: A dict of key:(index_memo, compression_parent, parents,
1944
opaque structure to pass to read_records to extract the raw
1947
Content that this record is built upon, may be None
1949
Logical parents of this node
1951
extra information about the content which needs to be passed to
1952
Factory.parse_record
1954
parent_map = self.get_parent_map(keys)
1957
if key not in parent_map:
1959
method = self.get_method(key)
1960
parents = parent_map[key]
1961
if method == 'fulltext':
1962
compression_parent = None
1964
compression_parent = parents[0]
1965
noeol = 'no-eol' in self.get_options(key)
1966
index_memo = self.get_position(key)
1967
result[key] = (index_memo, compression_parent,
1968
parents, (method, noeol))
1971
def get_method(self, key):
1972
"""Return compression method of specified key."""
1973
options = self.get_options(key)
1974
if 'fulltext' in options:
1976
elif 'line-delta' in options:
1979
raise errors.KnitIndexUnknownMethod(self, options)
1981
def get_options(self, key):
1982
"""Return a list representing options.
1986
prefix, suffix = self._split_key(key)
1987
self._load_prefixes([prefix])
1989
return self._kndx_cache[prefix][0][suffix][1]
1991
raise RevisionNotPresent(key, self)
1993
def get_parent_map(self, keys):
1994
"""Get a map of the parents of keys.
1996
:param keys: The keys to look up parents for.
1997
:return: A mapping from keys to parents. Absent keys are absent from
2000
# Parse what we need to up front, this potentially trades off I/O
2001
# locality (.kndx and .knit in the same block group for the same file
2002
# id) for less checking in inner loops.
2003
prefixes = set(key[:-1] for key in keys)
2004
self._load_prefixes(prefixes)
2009
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
2013
result[key] = tuple(prefix + (suffix,) for
2014
suffix in suffix_parents)
2017
def get_position(self, key):
2018
"""Return details needed to access the version.
2020
:return: a tuple (key, data position, size) to hand to the access
2021
logic to get the record.
2023
prefix, suffix = self._split_key(key)
2024
self._load_prefixes([prefix])
2025
entry = self._kndx_cache[prefix][0][suffix]
2026
return key, entry[2], entry[3]
2028
has_key = _mod_index._has_key_from_parent_map
2030
def _init_index(self, path, extra_lines=[]):
2031
"""Initialize an index."""
2033
sio.write(self.HEADER)
2034
sio.writelines(extra_lines)
2036
self._transport.put_file_non_atomic(path, sio,
2037
create_parent_dir=True)
2038
# self._create_parent_dir)
2039
# mode=self._file_mode,
2040
# dir_mode=self._dir_mode)
2043
"""Get all the keys in the collection.
2045
The keys are not ordered.
2048
# Identify all key prefixes.
2049
# XXX: A bit hacky, needs polish.
2050
if type(self._mapper) == ConstantMapper:
2054
for quoted_relpath in self._transport.iter_files_recursive():
2055
path, ext = os.path.splitext(quoted_relpath)
2057
prefixes = [self._mapper.unmap(path) for path in relpaths]
2058
self._load_prefixes(prefixes)
2059
for prefix in prefixes:
2060
for suffix in self._kndx_cache[prefix][1]:
2061
result.add(prefix + (suffix,))
2064
def _load_prefixes(self, prefixes):
2065
"""Load the indices for prefixes."""
2067
for prefix in prefixes:
2068
if prefix not in self._kndx_cache:
2069
# the load_data interface writes to these variables.
2072
self._filename = prefix
2074
path = self._mapper.map(prefix) + '.kndx'
2075
fp = self._transport.get(path)
2077
# _load_data may raise NoSuchFile if the target knit is
2079
_load_data(self, fp)
2082
self._kndx_cache[prefix] = (self._cache, self._history)
2087
self._kndx_cache[prefix] = ({}, [])
2088
if type(self._mapper) == ConstantMapper:
2089
# preserve behaviour for revisions.kndx etc.
2090
self._init_index(path)
2095
missing_keys = _mod_index._missing_keys_from_parent_map
2097
def _partition_keys(self, keys):
2098
"""Turn keys into a dict of prefix:suffix_list."""
2101
prefix_keys = result.setdefault(key[:-1], [])
2102
prefix_keys.append(key[-1])
2105
def _dictionary_compress(self, keys):
2106
"""Dictionary compress keys.
2108
:param keys: The keys to generate references to.
2109
:return: A string representation of keys. keys which are present are
2110
dictionary compressed, and others are emitted as fulltext with a
2116
prefix = keys[0][:-1]
2117
cache = self._kndx_cache[prefix][0]
2119
if key[:-1] != prefix:
2120
# kndx indices cannot refer across partitioned storage.
2121
raise ValueError("mismatched prefixes for %r" % keys)
2122
if key[-1] in cache:
2123
# -- inlined lookup() --
2124
result_list.append(str(cache[key[-1]][5]))
2125
# -- end lookup () --
2127
result_list.append('.' + key[-1])
2128
return ' '.join(result_list)
2130
def _reset_cache(self):
2131
# Possibly this should be a LRU cache. A dictionary from key_prefix to
2132
# (cache_dict, history_vector) for parsed kndx files.
2133
self._kndx_cache = {}
2134
self._scope = self._get_scope()
2135
allow_writes = self._allow_writes()
2141
def _sort_keys_by_io(self, keys, positions):
2142
"""Figure out an optimal order to read the records for the given keys.
2144
Sort keys, grouped by index and sorted by position.
2146
:param keys: A list of keys whose records we want to read. This will be
2148
:param positions: A dict, such as the one returned by
2149
_get_components_positions()
2152
def get_sort_key(key):
2153
index_memo = positions[key][1]
2154
# Group by prefix and position. index_memo[0] is the key, so it is
2155
# (file_id, revision_id) and we don't want to sort on revision_id,
2156
# index_memo[1] is the position, and index_memo[2] is the size,
2157
# which doesn't matter for the sort
2158
return index_memo[0][:-1], index_memo[1]
2159
return keys.sort(key=get_sort_key)
2161
def _split_key(self, key):
2162
"""Split key into a prefix and suffix."""
2163
return key[:-1], key[-1]
2166
class _KnitGraphIndex(object):
2167
"""A KnitVersionedFiles index layered on GraphIndex."""
2169
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2171
"""Construct a KnitGraphIndex on a graph_index.
2173
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2174
:param is_locked: A callback to check whether the object should answer
2176
:param deltas: Allow delta-compressed records.
2177
:param parents: If True, record knits parents, if not do not record
2179
:param add_callback: If not None, allow additions to the index and call
2180
this callback with a list of added GraphIndex nodes:
2181
[(node, value, node_refs), ...]
2182
:param is_locked: A callback, returns True if the index is locked and
2185
self._add_callback = add_callback
2186
self._graph_index = graph_index
2187
self._deltas = deltas
2188
self._parents = parents
2189
if deltas and not parents:
2190
# XXX: TODO: Delta tree and parent graph should be conceptually
2192
raise KnitCorrupt(self, "Cannot do delta compression without "
2194
self.has_graph = parents
2195
self._is_locked = is_locked
2198
return "%s(%r)" % (self.__class__.__name__, self._graph_index)
2200
def add_records(self, records, random_id=False):
2201
"""Add multiple records to the index.
2203
This function does not insert data into the Immutable GraphIndex
2204
backing the KnitGraphIndex, instead it prepares data for insertion by
2205
the caller and checks that it is safe to insert then calls
2206
self._add_callback with the prepared GraphIndex nodes.
2208
:param records: a list of tuples:
2209
(key, options, access_memo, parents).
2210
:param random_id: If True the ids being added were randomly generated
2211
and no check for existence will be performed.
2213
if not self._add_callback:
2214
raise errors.ReadOnlyError(self)
2215
# we hope there are no repositories with inconsistent parentage
2219
for (key, options, access_memo, parents) in records:
2221
parents = tuple(parents)
2222
index, pos, size = access_memo
2223
if 'no-eol' in options:
2227
value += "%d %d" % (pos, size)
2228
if not self._deltas:
2229
if 'line-delta' in options:
2230
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
2233
if 'line-delta' in options:
2234
node_refs = (parents, (parents[0],))
2236
node_refs = (parents, ())
2238
node_refs = (parents, )
2241
raise KnitCorrupt(self, "attempt to add node with parents "
2242
"in parentless index.")
2244
keys[key] = (value, node_refs)
2247
present_nodes = self._get_entries(keys)
2248
for (index, key, value, node_refs) in present_nodes:
2249
if (value[0] != keys[key][0][0] or
2250
node_refs != keys[key][1]):
2251
raise KnitCorrupt(self, "inconsistent details in add_records"
2252
": %s %s" % ((value, node_refs), keys[key]))
2256
for key, (value, node_refs) in keys.iteritems():
2257
result.append((key, value, node_refs))
2259
for key, (value, node_refs) in keys.iteritems():
2260
result.append((key, value))
2261
self._add_callback(result)
2263
def _check_read(self):
2264
"""raise if reads are not permitted."""
2265
if not self._is_locked():
2266
raise errors.ObjectNotLocked(self)
2268
def _check_write_ok(self):
2269
"""Assert if writes are not permitted."""
2270
if not self._is_locked():
2271
raise errors.ObjectNotLocked(self)
2273
def _compression_parent(self, an_entry):
2274
# return the key that an_entry is compressed against, or None
2275
# Grab the second parent list (as deltas implies parents currently)
2276
compression_parents = an_entry[3][1]
2277
if not compression_parents:
2279
if len(compression_parents) != 1:
2280
raise AssertionError(
2281
"Too many compression parents: %r" % compression_parents)
2282
return compression_parents[0]
2284
def get_build_details(self, keys):
2285
"""Get the method, index_memo and compression parent for version_ids.
2287
Ghosts are omitted from the result.
2289
:param keys: An iterable of keys.
2290
:return: A dict of key:
2291
(index_memo, compression_parent, parents, record_details).
2293
opaque structure to pass to read_records to extract the raw
2296
Content that this record is built upon, may be None
2298
Logical parents of this node
2300
extra information about the content which needs to be passed to
2301
Factory.parse_record
2305
entries = self._get_entries(keys, False)
2306
for entry in entries:
2308
if not self._parents:
2311
parents = entry[3][0]
2312
if not self._deltas:
2313
compression_parent_key = None
2315
compression_parent_key = self._compression_parent(entry)
2316
noeol = (entry[2][0] == 'N')
2317
if compression_parent_key:
2318
method = 'line-delta'
2321
result[key] = (self._node_to_position(entry),
2322
compression_parent_key, parents,
2326
def _get_entries(self, keys, check_present=False):
2327
"""Get the entries for keys.
2329
:param keys: An iterable of index key tuples.
2334
for node in self._graph_index.iter_entries(keys):
2336
found_keys.add(node[1])
2338
# adapt parentless index to the rest of the code.
2339
for node in self._graph_index.iter_entries(keys):
2340
yield node[0], node[1], node[2], ()
2341
found_keys.add(node[1])
2343
missing_keys = keys.difference(found_keys)
2345
raise RevisionNotPresent(missing_keys.pop(), self)
2347
def get_method(self, key):
2348
"""Return compression method of specified key."""
2349
return self._get_method(self._get_node(key))
2351
def _get_method(self, node):
2352
if not self._deltas:
2354
if self._compression_parent(node):
2359
def _get_node(self, key):
2361
return list(self._get_entries([key]))[0]
2363
raise RevisionNotPresent(key, self)
2365
def get_options(self, key):
2366
"""Return a list representing options.
2370
node = self._get_node(key)
2371
options = [self._get_method(node)]
2372
if node[2][0] == 'N':
2373
options.append('no-eol')
2376
def get_parent_map(self, keys):
2377
"""Get a map of the parents of keys.
2379
:param keys: The keys to look up parents for.
2380
:return: A mapping from keys to parents. Absent keys are absent from
2384
nodes = self._get_entries(keys)
2388
result[node[1]] = node[3][0]
2391
result[node[1]] = None
2394
def get_position(self, key):
2395
"""Return details needed to access the version.
2397
:return: a tuple (index, data position, size) to hand to the access
2398
logic to get the record.
2400
node = self._get_node(key)
2401
return self._node_to_position(node)
2403
has_key = _mod_index._has_key_from_parent_map
2406
"""Get all the keys in the collection.
2408
The keys are not ordered.
2411
return [node[1] for node in self._graph_index.iter_all_entries()]
2413
missing_keys = _mod_index._missing_keys_from_parent_map
2415
def _node_to_position(self, node):
2416
"""Convert an index value to position details."""
2417
bits = node[2][1:].split(' ')
2418
return node[0], int(bits[0]), int(bits[1])
2420
def _sort_keys_by_io(self, keys, positions):
2421
"""Figure out an optimal order to read the records for the given keys.
2423
Sort keys, grouped by index and sorted by position.
2425
:param keys: A list of keys whose records we want to read. This will be
2427
:param positions: A dict, such as the one returned by
2428
_get_components_positions()
2431
def get_index_memo(key):
2432
# index_memo is at offset [1]. It is made up of (GraphIndex,
2433
# position, size). GI is an object, which will be unique for each
2434
# pack file. This causes us to group by pack file, then sort by
2435
# position. Size doesn't matter, but it isn't worth breaking up the
2437
return positions[key][1]
2438
return keys.sort(key=get_index_memo)
2441
class _KnitKeyAccess(object):
2442
"""Access to records in .knit files."""
2444
def __init__(self, transport, mapper):
2445
"""Create a _KnitKeyAccess with transport and mapper.
2447
:param transport: The transport the access object is rooted at.
2448
:param mapper: The mapper used to map keys to .knit files.
2450
self._transport = transport
2451
self._mapper = mapper
2453
def add_raw_records(self, key_sizes, raw_data):
2454
"""Add raw knit bytes to a storage area.
2456
The data is spooled to the container writer in one bytes-record per
2459
:param sizes: An iterable of tuples containing the key and size of each
2461
:param raw_data: A bytestring containing the data.
2462
:return: A list of memos to retrieve the record later. Each memo is an
2463
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2464
length), where the key is the record key.
2466
if type(raw_data) != str:
2467
raise AssertionError(
2468
'data must be plain bytes was %s' % type(raw_data))
2471
# TODO: This can be tuned for writing to sftp and other servers where
2472
# append() is relatively expensive by grouping the writes to each key
2474
for key, size in key_sizes:
2475
path = self._mapper.map(key)
2477
base = self._transport.append_bytes(path + '.knit',
2478
raw_data[offset:offset+size])
2479
except errors.NoSuchFile:
2480
self._transport.mkdir(osutils.dirname(path))
2481
base = self._transport.append_bytes(path + '.knit',
2482
raw_data[offset:offset+size])
2486
result.append((key, base, size))
2489
def get_raw_records(self, memos_for_retrieval):
2490
"""Get the raw bytes for a records.
2492
:param memos_for_retrieval: An iterable containing the access memo for
2493
retrieving the bytes.
2494
:return: An iterator over the bytes of the records.
2496
# first pass, group into same-index request to minimise readv's issued.
2498
current_prefix = None
2499
for (key, offset, length) in memos_for_retrieval:
2500
if current_prefix == key[:-1]:
2501
current_list.append((offset, length))
2503
if current_prefix is not None:
2504
request_lists.append((current_prefix, current_list))
2505
current_prefix = key[:-1]
2506
current_list = [(offset, length)]
2507
# handle the last entry
2508
if current_prefix is not None:
2509
request_lists.append((current_prefix, current_list))
2510
for prefix, read_vector in request_lists:
2511
path = self._mapper.map(prefix) + '.knit'
2512
for pos, data in self._transport.readv(path, read_vector):
2516
class _DirectPackAccess(object):
2517
"""Access to data in one or more packs with less translation."""
2519
def __init__(self, index_to_packs, reload_func=None):
2520
"""Create a _DirectPackAccess object.
2522
:param index_to_packs: A dict mapping index objects to the transport
2523
and file names for obtaining data.
2524
:param reload_func: A function to call if we determine that the pack
2525
files have moved and we need to reload our caches. See
2526
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
2528
self._container_writer = None
2529
self._write_index = None
2530
self._indices = index_to_packs
2531
self._reload_func = reload_func
2533
def add_raw_records(self, key_sizes, raw_data):
2534
"""Add raw knit bytes to a storage area.
2536
The data is spooled to the container writer in one bytes-record per
2539
:param sizes: An iterable of tuples containing the key and size of each
2541
:param raw_data: A bytestring containing the data.
2542
:return: A list of memos to retrieve the record later. Each memo is an
2543
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2544
length), where the index field is the write_index object supplied
2545
to the PackAccess object.
2547
if type(raw_data) != str:
2548
raise AssertionError(
2549
'data must be plain bytes was %s' % type(raw_data))
2552
for key, size in key_sizes:
2553
p_offset, p_length = self._container_writer.add_bytes_record(
2554
raw_data[offset:offset+size], [])
2556
result.append((self._write_index, p_offset, p_length))
2559
def get_raw_records(self, memos_for_retrieval):
2560
"""Get the raw bytes for a records.
2562
:param memos_for_retrieval: An iterable containing the (index, pos,
2563
length) memo for retrieving the bytes. The Pack access method
2564
looks up the pack to use for a given record in its index_to_pack
2566
:return: An iterator over the bytes of the records.
2568
# first pass, group into same-index requests
2570
current_index = None
2571
for (index, offset, length) in memos_for_retrieval:
2572
if current_index == index:
2573
current_list.append((offset, length))
2575
if current_index is not None:
2576
request_lists.append((current_index, current_list))
2577
current_index = index
2578
current_list = [(offset, length)]
2579
# handle the last entry
2580
if current_index is not None:
2581
request_lists.append((current_index, current_list))
2582
for index, offsets in request_lists:
2584
transport, path = self._indices[index]
2586
# A KeyError here indicates that someone has triggered an index
2587
# reload, and this index has gone missing, we need to start
2589
if self._reload_func is None:
2590
# If we don't have a _reload_func there is nothing that can
2593
raise errors.RetryWithNewPacks(reload_occurred=True,
2594
exc_info=sys.exc_info())
2596
reader = pack.make_readv_reader(transport, path, offsets)
2597
for names, read_func in reader.iter_records():
2598
yield read_func(None)
2599
except errors.NoSuchFile:
2600
# A NoSuchFile error indicates that a pack file has gone
2601
# missing on disk, we need to trigger a reload, and start over.
2602
if self._reload_func is None:
2604
raise errors.RetryWithNewPacks(reload_occurred=False,
2605
exc_info=sys.exc_info())
2607
def set_writer(self, writer, index, transport_packname):
2608
"""Set a writer to use for adding data."""
2609
if index is not None:
2610
self._indices[index] = transport_packname
2611
self._container_writer = writer
2612
self._write_index = index
2614
def reload_or_raise(self, retry_exc):
2615
"""Try calling the reload function, or re-raise the original exception.
2617
This should be called after _DirectPackAccess raises a
2618
RetryWithNewPacks exception. This function will handle the common logic
2619
of determining when the error is fatal versus being temporary.
2620
It will also make sure that the original exception is raised, rather
2621
than the RetryWithNewPacks exception.
2623
If this function returns, then the calling function should retry
2624
whatever operation was being performed. Otherwise an exception will
2627
:param retry_exc: A RetryWithNewPacks exception.
2630
if self._reload_func is None:
2632
elif not self._reload_func():
2633
# The reload claimed that nothing changed
2634
if not retry_exc.reload_occurred:
2635
# If there wasn't an earlier reload, then we really were
2636
# expecting to find changes. We didn't find them, so this is a
2640
exc_class, exc_value, exc_traceback = retry_exc.exc_info
2641
raise exc_class, exc_value, exc_traceback
2644
# Deprecated, use PatienceSequenceMatcher instead
2645
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2648
def annotate_knit(knit, revision_id):
2649
"""Annotate a knit with no cached annotations.
2651
This implementation is for knits with no cached annotations.
2652
It will work for knits with cached annotations, but this is not
2655
annotator = _KnitAnnotator(knit)
2656
return iter(annotator.annotate(revision_id))
2659
class _KnitAnnotator(object):
2660
"""Build up the annotations for a text."""
2662
def __init__(self, knit):
2665
# Content objects, differs from fulltexts because of how final newlines
2666
# are treated by knits. the content objects here will always have a
2668
self._fulltext_contents = {}
2670
# Annotated lines of specific revisions
2671
self._annotated_lines = {}
2673
# Track the raw data for nodes that we could not process yet.
2674
# This maps the revision_id of the base to a list of children that will
2675
# annotated from it.
2676
self._pending_children = {}
2678
# Nodes which cannot be extracted
2679
self._ghosts = set()
2681
# Track how many children this node has, so we know if we need to keep
2683
self._annotate_children = {}
2684
self._compression_children = {}
2686
self._all_build_details = {}
2687
# The children => parent revision_id graph
2688
self._revision_id_graph = {}
2690
self._heads_provider = None
2692
self._nodes_to_keep_annotations = set()
2693
self._generations_until_keep = 100
2695
def set_generations_until_keep(self, value):
2696
"""Set the number of generations before caching a node.
2698
Setting this to -1 will cache every merge node, setting this higher
2699
will cache fewer nodes.
2701
self._generations_until_keep = value
2703
def _add_fulltext_content(self, revision_id, content_obj):
2704
self._fulltext_contents[revision_id] = content_obj
2705
# TODO: jam 20080305 It might be good to check the sha1digest here
2706
return content_obj.text()
2708
def _check_parents(self, child, nodes_to_annotate):
2709
"""Check if all parents have been processed.
2711
:param child: A tuple of (rev_id, parents, raw_content)
2712
:param nodes_to_annotate: If child is ready, add it to
2713
nodes_to_annotate, otherwise put it back in self._pending_children
2715
for parent_id in child[1]:
2716
if (parent_id not in self._annotated_lines):
2717
# This parent is present, but another parent is missing
2718
self._pending_children.setdefault(parent_id,
2722
# This one is ready to be processed
2723
nodes_to_annotate.append(child)
2725
def _add_annotation(self, revision_id, fulltext, parent_ids,
2726
left_matching_blocks=None):
2727
"""Add an annotation entry.
2729
All parents should already have been annotated.
2730
:return: A list of children that now have their parents satisfied.
2732
a = self._annotated_lines
2733
annotated_parent_lines = [a[p] for p in parent_ids]
2734
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2735
fulltext, revision_id, left_matching_blocks,
2736
heads_provider=self._get_heads_provider()))
2737
self._annotated_lines[revision_id] = annotated_lines
2738
for p in parent_ids:
2739
ann_children = self._annotate_children[p]
2740
ann_children.remove(revision_id)
2741
if (not ann_children
2742
and p not in self._nodes_to_keep_annotations):
2743
del self._annotated_lines[p]
2744
del self._all_build_details[p]
2745
if p in self._fulltext_contents:
2746
del self._fulltext_contents[p]
2747
# Now that we've added this one, see if there are any pending
2748
# deltas to be done, certainly this parent is finished
2749
nodes_to_annotate = []
2750
for child in self._pending_children.pop(revision_id, []):
2751
self._check_parents(child, nodes_to_annotate)
2752
return nodes_to_annotate
2754
def _get_build_graph(self, key):
2755
"""Get the graphs for building texts and annotations.
2757
The data you need for creating a full text may be different than the
2758
data you need to annotate that text. (At a minimum, you need both
2759
parents to create an annotation, but only need 1 parent to generate the
2762
:return: A list of (key, index_memo) records, suitable for
2763
passing to read_records_iter to start reading in the raw data fro/
2766
if key in self._annotated_lines:
2769
pending = set([key])
2774
# get all pending nodes
2776
this_iteration = pending
2777
build_details = self._knit._index.get_build_details(this_iteration)
2778
self._all_build_details.update(build_details)
2779
# new_nodes = self._knit._index._get_entries(this_iteration)
2781
for key, details in build_details.iteritems():
2782
(index_memo, compression_parent, parents,
2783
record_details) = details
2784
self._revision_id_graph[key] = parents
2785
records.append((key, index_memo))
2786
# Do we actually need to check _annotated_lines?
2787
pending.update(p for p in parents
2788
if p not in self._all_build_details)
2789
if compression_parent:
2790
self._compression_children.setdefault(compression_parent,
2793
for parent in parents:
2794
self._annotate_children.setdefault(parent,
2796
num_gens = generation - kept_generation
2797
if ((num_gens >= self._generations_until_keep)
2798
and len(parents) > 1):
2799
kept_generation = generation
2800
self._nodes_to_keep_annotations.add(key)
2802
missing_versions = this_iteration.difference(build_details.keys())
2803
self._ghosts.update(missing_versions)
2804
for missing_version in missing_versions:
2805
# add a key, no parents
2806
self._revision_id_graph[missing_version] = ()
2807
pending.discard(missing_version) # don't look for it
2808
if self._ghosts.intersection(self._compression_children):
2810
"We cannot have nodes which have a ghost compression parent:\n"
2812
"compression children: %r"
2813
% (self._ghosts, self._compression_children))
2814
# Cleanout anything that depends on a ghost so that we don't wait for
2815
# the ghost to show up
2816
for node in self._ghosts:
2817
if node in self._annotate_children:
2818
# We won't be building this node
2819
del self._annotate_children[node]
2820
# Generally we will want to read the records in reverse order, because
2821
# we find the parent nodes after the children
2825
def _annotate_records(self, records):
2826
"""Build the annotations for the listed records."""
2827
# We iterate in the order read, rather than a strict order requested
2828
# However, process what we can, and put off to the side things that
2829
# still need parents, cleaning them up when those parents are
2831
for (rev_id, record,
2832
digest) in self._knit._read_records_iter(records):
2833
if rev_id in self._annotated_lines:
2835
parent_ids = self._revision_id_graph[rev_id]
2836
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2837
details = self._all_build_details[rev_id]
2838
(index_memo, compression_parent, parents,
2839
record_details) = details
2840
nodes_to_annotate = []
2841
# TODO: Remove the punning between compression parents, and
2842
# parent_ids, we should be able to do this without assuming
2844
if len(parent_ids) == 0:
2845
# There are no parents for this node, so just add it
2846
# TODO: This probably needs to be decoupled
2847
fulltext_content, delta = self._knit._factory.parse_record(
2848
rev_id, record, record_details, None)
2849
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2850
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2851
parent_ids, left_matching_blocks=None))
2853
child = (rev_id, parent_ids, record)
2854
# Check if all the parents are present
2855
self._check_parents(child, nodes_to_annotate)
2856
while nodes_to_annotate:
2857
# Should we use a queue here instead of a stack?
2858
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2859
(index_memo, compression_parent, parents,
2860
record_details) = self._all_build_details[rev_id]
2862
if compression_parent is not None:
2863
comp_children = self._compression_children[compression_parent]
2864
if rev_id not in comp_children:
2865
raise AssertionError("%r not in compression children %r"
2866
% (rev_id, comp_children))
2867
# If there is only 1 child, it is safe to reuse this
2869
reuse_content = (len(comp_children) == 1
2870
and compression_parent not in
2871
self._nodes_to_keep_annotations)
2873
# Remove it from the cache since it will be changing
2874
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2875
# Make sure to copy the fulltext since it might be
2877
parent_fulltext = list(parent_fulltext_content.text())
2879
parent_fulltext_content = self._fulltext_contents[compression_parent]
2880
parent_fulltext = parent_fulltext_content.text()
2881
comp_children.remove(rev_id)
2882
fulltext_content, delta = self._knit._factory.parse_record(
2883
rev_id, record, record_details,
2884
parent_fulltext_content,
2885
copy_base_content=(not reuse_content))
2886
fulltext = self._add_fulltext_content(rev_id,
2888
if compression_parent == parent_ids[0]:
2889
# the compression_parent is the left parent, so we can
2891
blocks = KnitContent.get_line_delta_blocks(delta,
2892
parent_fulltext, fulltext)
2894
fulltext_content = self._knit._factory.parse_fulltext(
2896
fulltext = self._add_fulltext_content(rev_id,
2898
nodes_to_annotate.extend(
2899
self._add_annotation(rev_id, fulltext, parent_ids,
2900
left_matching_blocks=blocks))
2902
def _get_heads_provider(self):
2903
"""Create a heads provider for resolving ancestry issues."""
2904
if self._heads_provider is not None:
2905
return self._heads_provider
2906
parent_provider = _mod_graph.DictParentsProvider(
2907
self._revision_id_graph)
2908
graph_obj = _mod_graph.Graph(parent_provider)
2909
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2910
self._heads_provider = head_cache
2913
def annotate(self, key):
2914
"""Return the annotated fulltext at the given key.
2916
:param key: The key to annotate.
2918
if len(self._knit._fallback_vfs) > 0:
2919
# stacked knits can't use the fast path at present.
2920
return self._simple_annotate(key)
2923
records = self._get_build_graph(key)
2924
if key in self._ghosts:
2925
raise errors.RevisionNotPresent(key, self._knit)
2926
self._annotate_records(records)
2927
return self._annotated_lines[key]
2928
except errors.RetryWithNewPacks, e:
2929
self._knit._access.reload_or_raise(e)
2930
# The cached build_details are no longer valid
2931
self._all_build_details.clear()
2933
def _simple_annotate(self, key):
2934
"""Return annotated fulltext, rediffing from the full texts.
2936
This is slow but makes no assumptions about the repository
2937
being able to produce line deltas.
2939
# TODO: this code generates a parent maps of present ancestors; it
2940
# could be split out into a separate method, and probably should use
2941
# iter_ancestry instead. -- mbp and robertc 20080704
2942
graph = _mod_graph.Graph(self._knit)
2943
head_cache = _mod_graph.FrozenHeadsCache(graph)
2944
search = graph._make_breadth_first_searcher([key])
2948
present, ghosts = search.next_with_ghosts()
2949
except StopIteration:
2951
keys.update(present)
2952
parent_map = self._knit.get_parent_map(keys)
2954
reannotate = annotate.reannotate
2955
for record in self._knit.get_record_stream(keys, 'topological', True):
2957
fulltext = split_lines(record.get_bytes_as('fulltext'))
2958
parents = parent_map[key]
2959
if parents is not None:
2960
parent_lines = [parent_cache[parent] for parent in parent_map[key]]
2963
parent_cache[key] = list(
2964
reannotate(parent_lines, fulltext, key, None, head_cache))
2966
return parent_cache[key]
2968
raise errors.RevisionNotPresent(key, self._knit)
2972
from bzrlib._knit_load_data_c import _load_data_c as _load_data
2974
from bzrlib._knit_load_data_py import _load_data_py as _load_data
added
removed
bzrlib/knit.py
