1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Knit versionedfile implementation.
19
A knit is a versioned file implementation that supports efficient append only
23
lifeless: the data file is made up of "delta records". each delta record has a delta header
24
that contains; (1) a version id, (2) the size of the delta (in lines), and (3) the digest of
25
the -expanded data- (ie, the delta applied to the parent). the delta also ends with a
26
end-marker; simply "end VERSION"
28
delta can be line or full contents.a
29
... the 8's there are the index number of the annotation.
30
version robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad 7 c7d23b2a5bd6ca00e8e266cec0ec228158ee9f9e
34
8 e.set('executable', 'yes')
36
8 if elt.get('executable') == 'yes':
37
8 ie.executable = True
38
end robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad
42
09:33 < jrydberg> lifeless: each index is made up of a tuple of; version id, options, position, size, parents
43
09:33 < jrydberg> lifeless: the parents are currently dictionary compressed
44
09:33 < jrydberg> lifeless: (meaning it currently does not support ghosts)
45
09:33 < lifeless> right
46
09:33 < jrydberg> lifeless: the position and size is the range in the data file
49
so the index sequence is the dictionary compressed sequence number used
50
in the deltas to provide line annotation
55
# 10:16 < lifeless> make partial index writes safe
56
# 10:16 < lifeless> implement 'knit.check()' like weave.check()
57
# 10:17 < lifeless> record known ghosts so we can detect when they are filled in rather than the current 'reweave
59
# move sha1 out of the content so that join is faster at verifying parents
60
# record content length ?
63
from cStringIO import StringIO
64
from itertools import izip, chain
69
from bzrlib.lazy_import import lazy_import
70
lazy_import(globals(), """
90
from bzrlib.errors import (
98
RevisionAlreadyPresent,
101
from bzrlib.osutils import (
108
from bzrlib.versionedfile import (
109
AbsentContentFactory,
113
ChunkedContentFactory,
119
# TODO: Split out code specific to this format into an associated object.
121
# TODO: Can we put in some kind of value to check that the index and data
122
# files belong together?
124
# TODO: accommodate binaries, perhaps by storing a byte count
126
# TODO: function to check whole file
128
# TODO: atomically append data, then measure backwards from the cursor
129
# position after writing to work out where it was located. we may need to
130
# bypass python file buffering.
132
DATA_SUFFIX = '.knit'
133
INDEX_SUFFIX = '.kndx'
136
class KnitAdapter(object):
137
"""Base class for knit record adaption."""
139
def __init__(self, basis_vf):
140
"""Create an adapter which accesses full texts from basis_vf.
142
:param basis_vf: A versioned file to access basis texts of deltas from.
143
May be None for adapters that do not need to access basis texts.
145
self._data = KnitVersionedFiles(None, None)
146
self._annotate_factory = KnitAnnotateFactory()
147
self._plain_factory = KnitPlainFactory()
148
self._basis_vf = basis_vf
151
class FTAnnotatedToUnannotated(KnitAdapter):
152
"""An adapter from FT annotated knits to unannotated ones."""
154
def get_bytes(self, factory):
155
annotated_compressed_bytes = factory.get_bytes_as(factory.storage_kind)
157
self._data._parse_record_unchecked(annotated_compressed_bytes)
158
content = self._annotate_factory.parse_fulltext(contents, rec[1])
159
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
163
class DeltaAnnotatedToUnannotated(KnitAdapter):
164
"""An adapter for deltas from annotated to unannotated."""
166
def get_bytes(self, factory):
167
annotated_compressed_bytes = factory.get_bytes_as(factory.storage_kind)
169
self._data._parse_record_unchecked(annotated_compressed_bytes)
170
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
172
contents = self._plain_factory.lower_line_delta(delta)
173
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
177
class FTAnnotatedToFullText(KnitAdapter):
178
"""An adapter from FT annotated knits to unannotated ones."""
180
def get_bytes(self, factory):
181
annotated_compressed_bytes = factory.get_bytes_as(factory.storage_kind)
183
self._data._parse_record_unchecked(annotated_compressed_bytes)
184
content, delta = self._annotate_factory.parse_record(factory.key[-1],
185
contents, factory._build_details, None)
186
return ''.join(content.text())
189
class DeltaAnnotatedToFullText(KnitAdapter):
190
"""An adapter for deltas from annotated to unannotated."""
192
def get_bytes(self, factory):
193
annotated_compressed_bytes = factory.get_bytes_as(factory.storage_kind)
195
self._data._parse_record_unchecked(annotated_compressed_bytes)
196
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
198
compression_parent = factory.parents[0]
199
basis_entry = self._basis_vf.get_record_stream(
200
[compression_parent], 'unordered', True).next()
201
if basis_entry.storage_kind == 'absent':
202
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
203
basis_chunks = basis_entry.get_bytes_as('chunked')
204
basis_lines = osutils.chunks_to_lines(basis_chunks)
205
# Manually apply the delta because we have one annotated content and
207
basis_content = PlainKnitContent(basis_lines, compression_parent)
208
basis_content.apply_delta(delta, rec[1])
209
basis_content._should_strip_eol = factory._build_details[1]
210
return ''.join(basis_content.text())
213
class FTPlainToFullText(KnitAdapter):
214
"""An adapter from FT plain knits to unannotated ones."""
216
def get_bytes(self, factory):
217
compressed_bytes = factory.get_bytes_as(factory.storage_kind)
219
self._data._parse_record_unchecked(compressed_bytes)
220
content, delta = self._plain_factory.parse_record(factory.key[-1],
221
contents, factory._build_details, None)
222
return ''.join(content.text())
225
class DeltaPlainToFullText(KnitAdapter):
226
"""An adapter for deltas from annotated to unannotated."""
228
def get_bytes(self, factory):
229
compressed_bytes = factory.get_bytes_as(factory.storage_kind)
231
self._data._parse_record_unchecked(compressed_bytes)
232
delta = self._plain_factory.parse_line_delta(contents, rec[1])
233
compression_parent = factory.parents[0]
234
# XXX: string splitting overhead.
235
basis_entry = self._basis_vf.get_record_stream(
236
[compression_parent], 'unordered', True).next()
237
if basis_entry.storage_kind == 'absent':
238
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
239
basis_chunks = basis_entry.get_bytes_as('chunked')
240
basis_lines = osutils.chunks_to_lines(basis_chunks)
241
basis_content = PlainKnitContent(basis_lines, compression_parent)
242
# Manually apply the delta because we have one annotated content and
244
content, _ = self._plain_factory.parse_record(rec[1], contents,
245
factory._build_details, basis_content)
246
return ''.join(content.text())
249
class KnitContentFactory(ContentFactory):
250
"""Content factory for streaming from knits.
252
:seealso ContentFactory:
255
def __init__(self, key, parents, build_details, sha1, raw_record,
256
annotated, knit=None):
257
"""Create a KnitContentFactory for key.
260
:param parents: The parents.
261
:param build_details: The build details as returned from
263
:param sha1: The sha1 expected from the full text of this object.
264
:param raw_record: The bytes of the knit data from disk.
265
:param annotated: True if the raw data is annotated.
267
ContentFactory.__init__(self)
270
self.parents = parents
271
if build_details[0] == 'line-delta':
276
annotated_kind = 'annotated-'
279
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
280
self._raw_record = raw_record
281
self._build_details = build_details
284
def get_bytes_as(self, storage_kind):
285
if storage_kind == self.storage_kind:
286
return self._raw_record
287
if self._knit is not None:
288
if storage_kind == 'chunked':
289
return self._knit.get_lines(self.key[0])
290
elif storage_kind == 'fulltext':
291
return self._knit.get_text(self.key[0])
292
raise errors.UnavailableRepresentation(self.key, storage_kind,
296
class KnitContent(object):
297
"""Content of a knit version to which deltas can be applied.
299
This is always stored in memory as a list of lines with \n at the end,
300
plus a flag saying if the final ending is really there or not, because that
301
corresponds to the on-disk knit representation.
305
self._should_strip_eol = False
307
def apply_delta(self, delta, new_version_id):
308
"""Apply delta to this object to become new_version_id."""
309
raise NotImplementedError(self.apply_delta)
311
def line_delta_iter(self, new_lines):
312
"""Generate line-based delta from this content to new_lines."""
313
new_texts = new_lines.text()
314
old_texts = self.text()
315
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
316
for tag, i1, i2, j1, j2 in s.get_opcodes():
319
# ofrom, oto, length, data
320
yield i1, i2, j2 - j1, new_lines._lines[j1:j2]
322
def line_delta(self, new_lines):
323
return list(self.line_delta_iter(new_lines))
326
def get_line_delta_blocks(knit_delta, source, target):
327
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
328
target_len = len(target)
331
for s_begin, s_end, t_len, new_text in knit_delta:
332
true_n = s_begin - s_pos
335
# knit deltas do not provide reliable info about whether the
336
# last line of a file matches, due to eol handling.
337
if source[s_pos + n -1] != target[t_pos + n -1]:
340
yield s_pos, t_pos, n
341
t_pos += t_len + true_n
343
n = target_len - t_pos
345
if source[s_pos + n -1] != target[t_pos + n -1]:
348
yield s_pos, t_pos, n
349
yield s_pos + (target_len - t_pos), target_len, 0
352
class AnnotatedKnitContent(KnitContent):
353
"""Annotated content."""
355
def __init__(self, lines):
356
KnitContent.__init__(self)
360
"""Return a list of (origin, text) for each content line."""
361
lines = self._lines[:]
362
if self._should_strip_eol:
363
origin, last_line = lines[-1]
364
lines[-1] = (origin, last_line.rstrip('\n'))
367
def apply_delta(self, delta, new_version_id):
368
"""Apply delta to this object to become new_version_id."""
371
for start, end, count, delta_lines in delta:
372
lines[offset+start:offset+end] = delta_lines
373
offset = offset + (start - end) + count
377
lines = [text for origin, text in self._lines]
378
except ValueError, e:
379
# most commonly (only?) caused by the internal form of the knit
380
# missing annotation information because of a bug - see thread
382
raise KnitCorrupt(self,
383
"line in annotated knit missing annotation information: %s"
385
if self._should_strip_eol:
386
lines[-1] = lines[-1].rstrip('\n')
390
return AnnotatedKnitContent(self._lines[:])
393
class PlainKnitContent(KnitContent):
394
"""Unannotated content.
396
When annotate[_iter] is called on this content, the same version is reported
397
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
401
def __init__(self, lines, version_id):
402
KnitContent.__init__(self)
404
self._version_id = version_id
407
"""Return a list of (origin, text) for each content line."""
408
return [(self._version_id, line) for line in self._lines]
410
def apply_delta(self, delta, new_version_id):
411
"""Apply delta to this object to become new_version_id."""
414
for start, end, count, delta_lines in delta:
415
lines[offset+start:offset+end] = delta_lines
416
offset = offset + (start - end) + count
417
self._version_id = new_version_id
420
return PlainKnitContent(self._lines[:], self._version_id)
424
if self._should_strip_eol:
426
lines[-1] = lines[-1].rstrip('\n')
430
class _KnitFactory(object):
431
"""Base class for common Factory functions."""
433
def parse_record(self, version_id, record, record_details,
434
base_content, copy_base_content=True):
435
"""Parse a record into a full content object.
437
:param version_id: The official version id for this content
438
:param record: The data returned by read_records_iter()
439
:param record_details: Details about the record returned by
441
:param base_content: If get_build_details returns a compression_parent,
442
you must return a base_content here, else use None
443
:param copy_base_content: When building from the base_content, decide
444
you can either copy it and return a new object, or modify it in
446
:return: (content, delta) A Content object and possibly a line-delta,
449
method, noeol = record_details
450
if method == 'line-delta':
451
if copy_base_content:
452
content = base_content.copy()
454
content = base_content
455
delta = self.parse_line_delta(record, version_id)
456
content.apply_delta(delta, version_id)
458
content = self.parse_fulltext(record, version_id)
460
content._should_strip_eol = noeol
461
return (content, delta)
464
class KnitAnnotateFactory(_KnitFactory):
465
"""Factory for creating annotated Content objects."""
469
def make(self, lines, version_id):
470
num_lines = len(lines)
471
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
473
def parse_fulltext(self, content, version_id):
474
"""Convert fulltext to internal representation
476
fulltext content is of the format
477
revid(utf8) plaintext\n
478
internal representation is of the format:
481
# TODO: jam 20070209 The tests expect this to be returned as tuples,
482
# but the code itself doesn't really depend on that.
483
# Figure out a way to not require the overhead of turning the
484
# list back into tuples.
485
lines = [tuple(line.split(' ', 1)) for line in content]
486
return AnnotatedKnitContent(lines)
488
def parse_line_delta_iter(self, lines):
489
return iter(self.parse_line_delta(lines))
491
def parse_line_delta(self, lines, version_id, plain=False):
492
"""Convert a line based delta into internal representation.
494
line delta is in the form of:
495
intstart intend intcount
497
revid(utf8) newline\n
498
internal representation is
499
(start, end, count, [1..count tuples (revid, newline)])
501
:param plain: If True, the lines are returned as a plain
502
list without annotations, not as a list of (origin, content) tuples, i.e.
503
(start, end, count, [1..count newline])
510
def cache_and_return(line):
511
origin, text = line.split(' ', 1)
512
return cache.setdefault(origin, origin), text
514
# walk through the lines parsing.
515
# Note that the plain test is explicitly pulled out of the
516
# loop to minimise any performance impact
519
start, end, count = [int(n) for n in header.split(',')]
520
contents = [next().split(' ', 1)[1] for i in xrange(count)]
521
result.append((start, end, count, contents))
524
start, end, count = [int(n) for n in header.split(',')]
525
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
526
result.append((start, end, count, contents))
529
def get_fulltext_content(self, lines):
530
"""Extract just the content lines from a fulltext."""
531
return (line.split(' ', 1)[1] for line in lines)
533
def get_linedelta_content(self, lines):
534
"""Extract just the content from a line delta.
536
This doesn't return all of the extra information stored in a delta.
537
Only the actual content lines.
542
header = header.split(',')
543
count = int(header[2])
544
for i in xrange(count):
545
origin, text = next().split(' ', 1)
548
def lower_fulltext(self, content):
549
"""convert a fulltext content record into a serializable form.
551
see parse_fulltext which this inverts.
553
# TODO: jam 20070209 We only do the caching thing to make sure that
554
# the origin is a valid utf-8 line, eventually we could remove it
555
return ['%s %s' % (o, t) for o, t in content._lines]
557
def lower_line_delta(self, delta):
558
"""convert a delta into a serializable form.
560
See parse_line_delta which this inverts.
562
# TODO: jam 20070209 We only do the caching thing to make sure that
563
# the origin is a valid utf-8 line, eventually we could remove it
565
for start, end, c, lines in delta:
566
out.append('%d,%d,%d\n' % (start, end, c))
567
out.extend(origin + ' ' + text
568
for origin, text in lines)
571
def annotate(self, knit, key):
572
content = knit._get_content(key)
573
# adjust for the fact that serialised annotations are only key suffixes
575
if type(key) == tuple:
577
origins = content.annotate()
579
for origin, line in origins:
580
result.append((prefix + (origin,), line))
583
# XXX: This smells a bit. Why would key ever be a non-tuple here?
584
# Aren't keys defined to be tuples? -- spiv 20080618
585
return content.annotate()
588
class KnitPlainFactory(_KnitFactory):
589
"""Factory for creating plain Content objects."""
593
def make(self, lines, version_id):
594
return PlainKnitContent(lines, version_id)
596
def parse_fulltext(self, content, version_id):
597
"""This parses an unannotated fulltext.
599
Note that this is not a noop - the internal representation
600
has (versionid, line) - its just a constant versionid.
602
return self.make(content, version_id)
604
def parse_line_delta_iter(self, lines, version_id):
606
num_lines = len(lines)
607
while cur < num_lines:
610
start, end, c = [int(n) for n in header.split(',')]
611
yield start, end, c, lines[cur:cur+c]
614
def parse_line_delta(self, lines, version_id):
615
return list(self.parse_line_delta_iter(lines, version_id))
617
def get_fulltext_content(self, lines):
618
"""Extract just the content lines from a fulltext."""
621
def get_linedelta_content(self, lines):
622
"""Extract just the content from a line delta.
624
This doesn't return all of the extra information stored in a delta.
625
Only the actual content lines.
630
header = header.split(',')
631
count = int(header[2])
632
for i in xrange(count):
635
def lower_fulltext(self, content):
636
return content.text()
638
def lower_line_delta(self, delta):
640
for start, end, c, lines in delta:
641
out.append('%d,%d,%d\n' % (start, end, c))
645
def annotate(self, knit, key):
646
annotator = _KnitAnnotator(knit)
647
return annotator.annotate(key)
651
def make_file_factory(annotated, mapper):
652
"""Create a factory for creating a file based KnitVersionedFiles.
654
This is only functional enough to run interface tests, it doesn't try to
655
provide a full pack environment.
657
:param annotated: knit annotations are wanted.
658
:param mapper: The mapper from keys to paths.
660
def factory(transport):
661
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
662
access = _KnitKeyAccess(transport, mapper)
663
return KnitVersionedFiles(index, access, annotated=annotated)
667
def make_pack_factory(graph, delta, keylength):
668
"""Create a factory for creating a pack based VersionedFiles.
670
This is only functional enough to run interface tests, it doesn't try to
671
provide a full pack environment.
673
:param graph: Store a graph.
674
:param delta: Delta compress contents.
675
:param keylength: How long should keys be.
677
def factory(transport):
678
parents = graph or delta
684
max_delta_chain = 200
687
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
688
key_elements=keylength)
689
stream = transport.open_write_stream('newpack')
690
writer = pack.ContainerWriter(stream.write)
692
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
693
deltas=delta, add_callback=graph_index.add_nodes)
694
access = _DirectPackAccess({})
695
access.set_writer(writer, graph_index, (transport, 'newpack'))
696
result = KnitVersionedFiles(index, access,
697
max_delta_chain=max_delta_chain)
698
result.stream = stream
699
result.writer = writer
704
def cleanup_pack_knit(versioned_files):
705
versioned_files.stream.close()
706
versioned_files.writer.end()
709
class KnitVersionedFiles(VersionedFiles):
710
"""Storage for many versioned files using knit compression.
712
Backend storage is managed by indices and data objects.
714
:ivar _index: A _KnitGraphIndex or similar that can describe the
715
parents, graph, compression and data location of entries in this
716
KnitVersionedFiles. Note that this is only the index for
717
*this* vfs; if there are fallbacks they must be queried separately.
720
def __init__(self, index, data_access, max_delta_chain=200,
721
annotated=False, reload_func=None):
722
"""Create a KnitVersionedFiles with index and data_access.
724
:param index: The index for the knit data.
725
:param data_access: The access object to store and retrieve knit
727
:param max_delta_chain: The maximum number of deltas to permit during
728
insertion. Set to 0 to prohibit the use of deltas.
729
:param annotated: Set to True to cause annotations to be calculated and
730
stored during insertion.
731
:param reload_func: An function that can be called if we think we need
732
to reload the pack listing and try again. See
733
'bzrlib.repofmt.pack_repo.AggregateIndex' for the signature.
736
self._access = data_access
737
self._max_delta_chain = max_delta_chain
739
self._factory = KnitAnnotateFactory()
741
self._factory = KnitPlainFactory()
742
self._fallback_vfs = []
743
self._reload_func = reload_func
746
return "%s(%r, %r)" % (
747
self.__class__.__name__,
751
def add_fallback_versioned_files(self, a_versioned_files):
752
"""Add a source of texts for texts not present in this knit.
754
:param a_versioned_files: A VersionedFiles object.
756
self._fallback_vfs.append(a_versioned_files)
758
def add_lines(self, key, parents, lines, parent_texts=None,
759
left_matching_blocks=None, nostore_sha=None, random_id=False,
761
"""See VersionedFiles.add_lines()."""
762
self._index._check_write_ok()
763
self._check_add(key, lines, random_id, check_content)
765
# The caller might pass None if there is no graph data, but kndx
766
# indexes can't directly store that, so we give them
767
# an empty tuple instead.
769
return self._add(key, lines, parents,
770
parent_texts, left_matching_blocks, nostore_sha, random_id)
772
def _add(self, key, lines, parents, parent_texts,
773
left_matching_blocks, nostore_sha, random_id):
774
"""Add a set of lines on top of version specified by parents.
776
Any versions not present will be converted into ghosts.
778
# first thing, if the content is something we don't need to store, find
780
line_bytes = ''.join(lines)
781
digest = sha_string(line_bytes)
782
if nostore_sha == digest:
783
raise errors.ExistingContent
786
if parent_texts is None:
788
# Do a single query to ascertain parent presence; we only compress
789
# against parents in the same kvf.
790
present_parent_map = self._index.get_parent_map(parents)
791
for parent in parents:
792
if parent in present_parent_map:
793
present_parents.append(parent)
795
# Currently we can only compress against the left most present parent.
796
if (len(present_parents) == 0 or
797
present_parents[0] != parents[0]):
800
# To speed the extract of texts the delta chain is limited
801
# to a fixed number of deltas. This should minimize both
802
# I/O and the time spend applying deltas.
803
delta = self._check_should_delta(present_parents[0])
805
text_length = len(line_bytes)
808
if lines[-1][-1] != '\n':
809
# copy the contents of lines.
811
options.append('no-eol')
812
lines[-1] = lines[-1] + '\n'
816
if type(element) != str:
817
raise TypeError("key contains non-strings: %r" % (key,))
818
# Knit hunks are still last-element only
820
content = self._factory.make(lines, version_id)
821
if 'no-eol' in options:
822
# Hint to the content object that its text() call should strip the
824
content._should_strip_eol = True
825
if delta or (self._factory.annotated and len(present_parents) > 0):
826
# Merge annotations from parent texts if needed.
827
delta_hunks = self._merge_annotations(content, present_parents,
828
parent_texts, delta, self._factory.annotated,
829
left_matching_blocks)
832
options.append('line-delta')
833
store_lines = self._factory.lower_line_delta(delta_hunks)
834
size, bytes = self._record_to_data(key, digest,
837
options.append('fulltext')
838
# isinstance is slower and we have no hierarchy.
839
if self._factory.__class__ == KnitPlainFactory:
840
# Use the already joined bytes saving iteration time in
842
size, bytes = self._record_to_data(key, digest,
845
# get mixed annotation + content and feed it into the
847
store_lines = self._factory.lower_fulltext(content)
848
size, bytes = self._record_to_data(key, digest,
851
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
852
self._index.add_records(
853
((key, options, access_memo, parents),),
855
return digest, text_length, content
857
def annotate(self, key):
858
"""See VersionedFiles.annotate."""
859
return self._factory.annotate(self, key)
861
def check(self, progress_bar=None):
862
"""See VersionedFiles.check()."""
863
# This doesn't actually test extraction of everything, but that will
864
# impact 'bzr check' substantially, and needs to be integrated with
865
# care. However, it does check for the obvious problem of a delta with
867
keys = self._index.keys()
868
parent_map = self.get_parent_map(keys)
870
if self._index.get_method(key) != 'fulltext':
871
compression_parent = parent_map[key][0]
872
if compression_parent not in parent_map:
873
raise errors.KnitCorrupt(self,
874
"Missing basis parent %s for %s" % (
875
compression_parent, key))
876
for fallback_vfs in self._fallback_vfs:
879
def _check_add(self, key, lines, random_id, check_content):
880
"""check that version_id and lines are safe to add."""
882
if contains_whitespace(version_id):
883
raise InvalidRevisionId(version_id, self)
884
self.check_not_reserved_id(version_id)
885
# TODO: If random_id==False and the key is already present, we should
886
# probably check that the existing content is identical to what is
887
# being inserted, and otherwise raise an exception. This would make
888
# the bundle code simpler.
890
self._check_lines_not_unicode(lines)
891
self._check_lines_are_lines(lines)
893
def _check_header(self, key, line):
894
rec = self._split_header(line)
895
self._check_header_version(rec, key[-1])
898
def _check_header_version(self, rec, version_id):
899
"""Checks the header version on original format knit records.
901
These have the last component of the key embedded in the record.
903
if rec[1] != version_id:
904
raise KnitCorrupt(self,
905
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
907
def _check_should_delta(self, parent):
908
"""Iterate back through the parent listing, looking for a fulltext.
910
This is used when we want to decide whether to add a delta or a new
911
fulltext. It searches for _max_delta_chain parents. When it finds a
912
fulltext parent, it sees if the total size of the deltas leading up to
913
it is large enough to indicate that we want a new full text anyway.
915
Return True if we should create a new delta, False if we should use a
920
for count in xrange(self._max_delta_chain):
922
# Note that this only looks in the index of this particular
923
# KnitVersionedFiles, not in the fallbacks. This ensures that
924
# we won't store a delta spanning physical repository
926
build_details = self._index.get_build_details([parent])
927
parent_details = build_details[parent]
928
except (RevisionNotPresent, KeyError), e:
929
# Some basis is not locally present: always fulltext
931
index_memo, compression_parent, _, _ = parent_details
932
_, _, size = index_memo
933
if compression_parent is None:
937
# We don't explicitly check for presence because this is in an
938
# inner loop, and if it's missing it'll fail anyhow.
939
parent = compression_parent
941
# We couldn't find a fulltext, so we must create a new one
943
# Simple heuristic - if the total I/O wold be greater as a delta than
944
# the originally installed fulltext, we create a new fulltext.
945
return fulltext_size > delta_size
947
def _build_details_to_components(self, build_details):
948
"""Convert a build_details tuple to a position tuple."""
949
# record_details, access_memo, compression_parent
950
return build_details[3], build_details[0], build_details[1]
952
def _get_components_positions(self, keys, allow_missing=False):
953
"""Produce a map of position data for the components of keys.
955
This data is intended to be used for retrieving the knit records.
957
A dict of key to (record_details, index_memo, next, parents) is
959
method is the way referenced data should be applied.
960
index_memo is the handle to pass to the data access to actually get the
962
next is the build-parent of the version, or None for fulltexts.
963
parents is the version_ids of the parents of this version
965
:param allow_missing: If True do not raise an error on a missing component,
969
pending_components = keys
970
while pending_components:
971
build_details = self._index.get_build_details(pending_components)
972
current_components = set(pending_components)
973
pending_components = set()
974
for key, details in build_details.iteritems():
975
(index_memo, compression_parent, parents,
976
record_details) = details
977
method = record_details[0]
978
if compression_parent is not None:
979
pending_components.add(compression_parent)
980
component_data[key] = self._build_details_to_components(details)
981
missing = current_components.difference(build_details)
982
if missing and not allow_missing:
983
raise errors.RevisionNotPresent(missing.pop(), self)
984
return component_data
986
def _get_content(self, key, parent_texts={}):
987
"""Returns a content object that makes up the specified
989
cached_version = parent_texts.get(key, None)
990
if cached_version is not None:
991
# Ensure the cache dict is valid.
992
if not self.get_parent_map([key]):
993
raise RevisionNotPresent(key, self)
994
return cached_version
995
text_map, contents_map = self._get_content_maps([key])
996
return contents_map[key]
998
def _get_content_maps(self, keys, nonlocal_keys=None):
999
"""Produce maps of text and KnitContents
1001
:param keys: The keys to produce content maps for.
1002
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
1003
which are known to not be in this knit, but rather in one of the
1005
:return: (text_map, content_map) where text_map contains the texts for
1006
the requested versions and content_map contains the KnitContents.
1008
# FUTURE: This function could be improved for the 'extract many' case
1009
# by tracking each component and only doing the copy when the number of
1010
# children than need to apply delta's to it is > 1 or it is part of the
1013
multiple_versions = len(keys) != 1
1014
record_map = self._get_record_map(keys, allow_missing=True)
1019
if nonlocal_keys is None:
1020
nonlocal_keys = set()
1022
nonlocal_keys = frozenset(nonlocal_keys)
1023
missing_keys = set(nonlocal_keys)
1024
for source in self._fallback_vfs:
1025
if not missing_keys:
1027
for record in source.get_record_stream(missing_keys,
1029
if record.storage_kind == 'absent':
1031
missing_keys.remove(record.key)
1032
lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
1033
text_map[record.key] = lines
1034
content_map[record.key] = PlainKnitContent(lines, record.key)
1035
if record.key in keys:
1036
final_content[record.key] = content_map[record.key]
1038
if key in nonlocal_keys:
1043
while cursor is not None:
1045
record, record_details, digest, next = record_map[cursor]
1047
raise RevisionNotPresent(cursor, self)
1048
components.append((cursor, record, record_details, digest))
1050
if cursor in content_map:
1051
# no need to plan further back
1052
components.append((cursor, None, None, None))
1056
for (component_id, record, record_details,
1057
digest) in reversed(components):
1058
if component_id in content_map:
1059
content = content_map[component_id]
1061
content, delta = self._factory.parse_record(key[-1],
1062
record, record_details, content,
1063
copy_base_content=multiple_versions)
1064
if multiple_versions:
1065
content_map[component_id] = content
1067
final_content[key] = content
1069
# digest here is the digest from the last applied component.
1070
text = content.text()
1071
actual_sha = sha_strings(text)
1072
if actual_sha != digest:
1073
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
1074
text_map[key] = text
1075
return text_map, final_content
1077
def get_parent_map(self, keys):
1078
"""Get a map of the graph parents of keys.
1080
:param keys: The keys to look up parents for.
1081
:return: A mapping from keys to parents. Absent keys are absent from
1084
return self._get_parent_map_with_sources(keys)[0]
1086
def _get_parent_map_with_sources(self, keys):
1087
"""Get a map of the parents of keys.
1089
:param keys: The keys to look up parents for.
1090
:return: A tuple. The first element is a mapping from keys to parents.
1091
Absent keys are absent from the mapping. The second element is a
1092
list with the locations each key was found in. The first element
1093
is the in-this-knit parents, the second the first fallback source,
1097
sources = [self._index] + self._fallback_vfs
1100
for source in sources:
1103
new_result = source.get_parent_map(missing)
1104
source_results.append(new_result)
1105
result.update(new_result)
1106
missing.difference_update(set(new_result))
1107
return result, source_results
1109
def _get_record_map(self, keys, allow_missing=False):
1110
"""Produce a dictionary of knit records.
1112
:return: {key:(record, record_details, digest, next)}
1114
data returned from read_records
1116
opaque information to pass to parse_record
1118
SHA1 digest of the full text after all steps are done
1120
build-parent of the version, i.e. the leftmost ancestor.
1121
Will be None if the record is not a delta.
1122
:param keys: The keys to build a map for
1123
:param allow_missing: If some records are missing, rather than
1124
error, just return the data that could be generated.
1126
# This retries the whole request if anything fails. Potentially we
1127
# could be a bit more selective. We could track the keys whose records
1128
# we have successfully found, and then only request the new records
1129
# from there. However, _get_components_positions grabs the whole build
1130
# chain, which means we'll likely try to grab the same records again
1131
# anyway. Also, can the build chains change as part of a pack
1132
# operation? We wouldn't want to end up with a broken chain.
1135
position_map = self._get_components_positions(keys,
1136
allow_missing=allow_missing)
1137
# key = component_id, r = record_details, i_m = index_memo,
1139
records = [(key, i_m) for key, (r, i_m, n)
1140
in position_map.iteritems()]
1142
for key, record, digest in self._read_records_iter(records):
1143
(record_details, index_memo, next) = position_map[key]
1144
record_map[key] = record, record_details, digest, next
1146
except errors.RetryWithNewPacks, e:
1147
self._access.reload_or_raise(e)
1149
def _split_by_prefix(self, keys):
1150
"""For the given keys, split them up based on their prefix.
1152
To keep memory pressure somewhat under control, split the
1153
requests back into per-file-id requests, otherwise "bzr co"
1154
extracts the full tree into memory before writing it to disk.
1155
This should be revisited if _get_content_maps() can ever cross
1158
:param keys: An iterable of key tuples
1159
:return: A dict of {prefix: [key_list]}
1161
split_by_prefix = {}
1164
split_by_prefix.setdefault('', []).append(key)
1166
split_by_prefix.setdefault(key[0], []).append(key)
1167
return split_by_prefix
1169
def get_record_stream(self, keys, ordering, include_delta_closure):
1170
"""Get a stream of records for keys.
1172
:param keys: The keys to include.
1173
:param ordering: Either 'unordered' or 'topological'. A topologically
1174
sorted stream has compression parents strictly before their
1176
:param include_delta_closure: If True then the closure across any
1177
compression parents will be included (in the opaque data).
1178
:return: An iterator of ContentFactory objects, each of which is only
1179
valid until the iterator is advanced.
1181
# keys might be a generator
1185
if not self._index.has_graph:
1186
# Cannot topological order when no graph has been stored.
1187
ordering = 'unordered'
1189
remaining_keys = keys
1192
keys = set(remaining_keys)
1193
for content_factory in self._get_remaining_record_stream(keys,
1194
ordering, include_delta_closure):
1195
remaining_keys.discard(content_factory.key)
1196
yield content_factory
1198
except errors.RetryWithNewPacks, e:
1199
self._access.reload_or_raise(e)
1201
def _get_remaining_record_stream(self, keys, ordering,
1202
include_delta_closure):
1203
"""This function is the 'retry' portion for get_record_stream."""
1204
if include_delta_closure:
1205
positions = self._get_components_positions(keys, allow_missing=True)
1207
build_details = self._index.get_build_details(keys)
1209
# (record_details, access_memo, compression_parent_key)
1210
positions = dict((key, self._build_details_to_components(details))
1211
for key, details in build_details.iteritems())
1212
absent_keys = keys.difference(set(positions))
1213
# There may be more absent keys : if we're missing the basis component
1214
# and are trying to include the delta closure.
1215
if include_delta_closure:
1216
needed_from_fallback = set()
1217
# Build up reconstructable_keys dict. key:True in this dict means
1218
# the key can be reconstructed.
1219
reconstructable_keys = {}
1223
chain = [key, positions[key][2]]
1225
needed_from_fallback.add(key)
1228
while chain[-1] is not None:
1229
if chain[-1] in reconstructable_keys:
1230
result = reconstructable_keys[chain[-1]]
1234
chain.append(positions[chain[-1]][2])
1236
# missing basis component
1237
needed_from_fallback.add(chain[-1])
1240
for chain_key in chain[:-1]:
1241
reconstructable_keys[chain_key] = result
1243
needed_from_fallback.add(key)
1244
# Double index lookups here : need a unified api ?
1245
global_map, parent_maps = self._get_parent_map_with_sources(keys)
1246
if ordering == 'topological':
1247
# Global topological sort
1248
present_keys = tsort.topo_sort(global_map)
1249
# Now group by source:
1251
current_source = None
1252
for key in present_keys:
1253
for parent_map in parent_maps:
1254
if key in parent_map:
1255
key_source = parent_map
1257
if current_source is not key_source:
1258
source_keys.append((key_source, []))
1259
current_source = key_source
1260
source_keys[-1][1].append(key)
1262
if ordering != 'unordered':
1263
raise AssertionError('valid values for ordering are:'
1264
' "unordered" or "topological" not: %r'
1266
# Just group by source; remote sources first.
1269
for parent_map in reversed(parent_maps):
1270
source_keys.append((parent_map, []))
1271
for key in parent_map:
1272
present_keys.append(key)
1273
source_keys[-1][1].append(key)
1274
# We have been requested to return these records in an order that
1275
# suits us. So we ask the index to give us an optimally sorted
1277
for source, sub_keys in source_keys:
1278
if source is parent_maps[0]:
1279
# Only sort the keys for this VF
1280
self._index._sort_keys_by_io(sub_keys, positions)
1281
absent_keys = keys - set(global_map)
1282
for key in absent_keys:
1283
yield AbsentContentFactory(key)
1284
# restrict our view to the keys we can answer.
1285
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1286
# XXX: At that point we need to consider the impact of double reads by
1287
# utilising components multiple times.
1288
if include_delta_closure:
1289
# XXX: get_content_maps performs its own index queries; allow state
1291
non_local_keys = needed_from_fallback - absent_keys
1292
prefix_split_keys = self._split_by_prefix(present_keys)
1293
prefix_split_non_local_keys = self._split_by_prefix(non_local_keys)
1294
for prefix, keys in prefix_split_keys.iteritems():
1295
non_local = prefix_split_non_local_keys.get(prefix, [])
1296
non_local = set(non_local)
1297
text_map, _ = self._get_content_maps(keys, non_local)
1299
lines = text_map.pop(key)
1300
yield ChunkedContentFactory(key, global_map[key], None,
1303
for source, keys in source_keys:
1304
if source is parent_maps[0]:
1305
# this KnitVersionedFiles
1306
records = [(key, positions[key][1]) for key in keys]
1307
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1308
(record_details, index_memo, _) = positions[key]
1309
yield KnitContentFactory(key, global_map[key],
1310
record_details, sha1, raw_data, self._factory.annotated, None)
1312
vf = self._fallback_vfs[parent_maps.index(source) - 1]
1313
for record in vf.get_record_stream(keys, ordering,
1314
include_delta_closure):
1317
def get_sha1s(self, keys):
1318
"""See VersionedFiles.get_sha1s()."""
1320
record_map = self._get_record_map(missing, allow_missing=True)
1322
for key, details in record_map.iteritems():
1323
if key not in missing:
1325
# record entry 2 is the 'digest'.
1326
result[key] = details[2]
1327
missing.difference_update(set(result))
1328
for source in self._fallback_vfs:
1331
new_result = source.get_sha1s(missing)
1332
result.update(new_result)
1333
missing.difference_update(set(new_result))
1336
def insert_record_stream(self, stream):
1337
"""Insert a record stream into this container.
1339
:param stream: A stream of records to insert.
1341
:seealso VersionedFiles.get_record_stream:
1343
def get_adapter(adapter_key):
1345
return adapters[adapter_key]
1347
adapter_factory = adapter_registry.get(adapter_key)
1348
adapter = adapter_factory(self)
1349
adapters[adapter_key] = adapter
1352
if self._factory.annotated:
1353
# self is annotated, we need annotated knits to use directly.
1354
annotated = "annotated-"
1357
# self is not annotated, but we can strip annotations cheaply.
1359
convertibles = set(["knit-annotated-ft-gz"])
1360
if self._max_delta_chain:
1361
delta_types.add("knit-annotated-delta-gz")
1362
convertibles.add("knit-annotated-delta-gz")
1363
# The set of types we can cheaply adapt without needing basis texts.
1364
native_types = set()
1365
if self._max_delta_chain:
1366
native_types.add("knit-%sdelta-gz" % annotated)
1367
delta_types.add("knit-%sdelta-gz" % annotated)
1368
native_types.add("knit-%sft-gz" % annotated)
1369
knit_types = native_types.union(convertibles)
1371
# Buffer all index entries that we can't add immediately because their
1372
# basis parent is missing. We don't buffer all because generating
1373
# annotations may require access to some of the new records. However we
1374
# can't generate annotations from new deltas until their basis parent
1375
# is present anyway, so we get away with not needing an index that
1376
# includes the new keys.
1378
# See <http://launchpad.net/bugs/300177> about ordering of compression
1379
# parents in the records - to be conservative, we insist that all
1380
# parents must be present to avoid expanding to a fulltext.
1382
# key = basis_parent, value = index entry to add
1383
buffered_index_entries = {}
1384
for record in stream:
1385
parents = record.parents
1386
if record.storage_kind in delta_types:
1387
# TODO: eventually the record itself should track
1388
# compression_parent
1389
compression_parent = parents[0]
1391
compression_parent = None
1392
# Raise an error when a record is missing.
1393
if record.storage_kind == 'absent':
1394
raise RevisionNotPresent([record.key], self)
1395
elif ((record.storage_kind in knit_types)
1396
and (compression_parent is None
1397
or not self._fallback_vfs
1398
or self._index.has_key(compression_parent)
1399
or not self.has_key(compression_parent))):
1400
# we can insert the knit record literally if either it has no
1401
# compression parent OR we already have its basis in this kvf
1402
# OR the basis is not present even in the fallbacks. In the
1403
# last case it will either turn up later in the stream and all
1404
# will be well, or it won't turn up at all and we'll raise an
1407
# TODO: self.has_key is somewhat redundant with
1408
# self._index.has_key; we really want something that directly
1409
# asks if it's only present in the fallbacks. -- mbp 20081119
1410
if record.storage_kind not in native_types:
1412
adapter_key = (record.storage_kind, "knit-delta-gz")
1413
adapter = get_adapter(adapter_key)
1415
adapter_key = (record.storage_kind, "knit-ft-gz")
1416
adapter = get_adapter(adapter_key)
1417
bytes = adapter.get_bytes(record)
1419
bytes = record.get_bytes_as(record.storage_kind)
1420
options = [record._build_details[0]]
1421
if record._build_details[1]:
1422
options.append('no-eol')
1423
# Just blat it across.
1424
# Note: This does end up adding data on duplicate keys. As
1425
# modern repositories use atomic insertions this should not
1426
# lead to excessive growth in the event of interrupted fetches.
1427
# 'knit' repositories may suffer excessive growth, but as a
1428
# deprecated format this is tolerable. It can be fixed if
1429
# needed by in the kndx index support raising on a duplicate
1430
# add with identical parents and options.
1431
access_memo = self._access.add_raw_records(
1432
[(record.key, len(bytes))], bytes)[0]
1433
index_entry = (record.key, options, access_memo, parents)
1435
if 'fulltext' not in options:
1436
# Not a fulltext, so we need to make sure the compression
1437
# parent will also be present.
1438
# Note that pack backed knits don't need to buffer here
1439
# because they buffer all writes to the transaction level,
1440
# but we don't expose that difference at the index level. If
1441
# the query here has sufficient cost to show up in
1442
# profiling we should do that.
1444
# They're required to be physically in this
1445
# KnitVersionedFiles, not in a fallback.
1446
if not self._index.has_key(compression_parent):
1447
pending = buffered_index_entries.setdefault(
1448
compression_parent, [])
1449
pending.append(index_entry)
1452
self._index.add_records([index_entry])
1453
elif record.storage_kind == 'chunked':
1454
self.add_lines(record.key, parents,
1455
osutils.chunks_to_lines(record.get_bytes_as('chunked')))
1456
elif record.storage_kind == 'fulltext':
1457
self.add_lines(record.key, parents,
1458
split_lines(record.get_bytes_as('fulltext')))
1460
# Not a fulltext, and not suitable for direct insertion as a
1461
# delta, either because it's not the right format, or this
1462
# KnitVersionedFiles doesn't permit deltas (_max_delta_chain ==
1463
# 0) or because it depends on a base only present in the
1465
adapter_key = record.storage_kind, 'fulltext'
1466
adapter = get_adapter(adapter_key)
1467
lines = split_lines(adapter.get_bytes(record))
1469
self.add_lines(record.key, parents, lines)
1470
except errors.RevisionAlreadyPresent:
1472
# Add any records whose basis parent is now available.
1473
added_keys = [record.key]
1475
key = added_keys.pop(0)
1476
if key in buffered_index_entries:
1477
index_entries = buffered_index_entries[key]
1478
self._index.add_records(index_entries)
1480
[index_entry[0] for index_entry in index_entries])
1481
del buffered_index_entries[key]
1482
# If there were any deltas which had a missing basis parent, error.
1483
if buffered_index_entries:
1484
from pprint import pformat
1485
raise errors.BzrCheckError(
1486
"record_stream refers to compression parents not in %r:\n%s"
1487
% (self, pformat(sorted(buffered_index_entries.keys()))))
1489
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1490
"""Iterate over the lines in the versioned files from keys.
1492
This may return lines from other keys. Each item the returned
1493
iterator yields is a tuple of a line and a text version that that line
1494
is present in (not introduced in).
1496
Ordering of results is in whatever order is most suitable for the
1497
underlying storage format.
1499
If a progress bar is supplied, it may be used to indicate progress.
1500
The caller is responsible for cleaning up progress bars (because this
1504
* Lines are normalised by the underlying store: they will all have \\n
1506
* Lines are returned in arbitrary order.
1507
* If a requested key did not change any lines (or didn't have any
1508
lines), it may not be mentioned at all in the result.
1510
:return: An iterator over (line, key).
1513
pb = progress.DummyProgress()
1519
# we don't care about inclusions, the caller cares.
1520
# but we need to setup a list of records to visit.
1521
# we need key, position, length
1523
build_details = self._index.get_build_details(keys)
1524
for key, details in build_details.iteritems():
1526
key_records.append((key, details[0]))
1527
records_iter = enumerate(self._read_records_iter(key_records))
1528
for (key_idx, (key, data, sha_value)) in records_iter:
1529
pb.update('Walking content.', key_idx, total)
1530
compression_parent = build_details[key][1]
1531
if compression_parent is None:
1533
line_iterator = self._factory.get_fulltext_content(data)
1536
line_iterator = self._factory.get_linedelta_content(data)
1537
# Now that we are yielding the data for this key, remove it
1540
# XXX: It might be more efficient to yield (key,
1541
# line_iterator) in the future. However for now, this is a
1542
# simpler change to integrate into the rest of the
1543
# codebase. RBC 20071110
1544
for line in line_iterator:
1547
except errors.RetryWithNewPacks, e:
1548
self._access.reload_or_raise(e)
1549
# If there are still keys we've not yet found, we look in the fallback
1550
# vfs, and hope to find them there. Note that if the keys are found
1551
# but had no changes or no content, the fallback may not return
1553
if keys and not self._fallback_vfs:
1554
# XXX: strictly the second parameter is meant to be the file id
1555
# but it's not easily accessible here.
1556
raise RevisionNotPresent(keys, repr(self))
1557
for source in self._fallback_vfs:
1561
for line, key in source.iter_lines_added_or_present_in_keys(keys):
1562
source_keys.add(key)
1564
keys.difference_update(source_keys)
1565
pb.update('Walking content.', total, total)
1567
def _make_line_delta(self, delta_seq, new_content):
1568
"""Generate a line delta from delta_seq and new_content."""
1570
for op in delta_seq.get_opcodes():
1571
if op[0] == 'equal':
1573
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1576
def _merge_annotations(self, content, parents, parent_texts={},
1577
delta=None, annotated=None,
1578
left_matching_blocks=None):
1579
"""Merge annotations for content and generate deltas.
1581
This is done by comparing the annotations based on changes to the text
1582
and generating a delta on the resulting full texts. If annotations are
1583
not being created then a simple delta is created.
1585
if left_matching_blocks is not None:
1586
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1590
for parent_key in parents:
1591
merge_content = self._get_content(parent_key, parent_texts)
1592
if (parent_key == parents[0] and delta_seq is not None):
1595
seq = patiencediff.PatienceSequenceMatcher(
1596
None, merge_content.text(), content.text())
1597
for i, j, n in seq.get_matching_blocks():
1600
# this copies (origin, text) pairs across to the new
1601
# content for any line that matches the last-checked
1603
content._lines[j:j+n] = merge_content._lines[i:i+n]
1604
# XXX: Robert says the following block is a workaround for a
1605
# now-fixed bug and it can probably be deleted. -- mbp 20080618
1606
if content._lines and content._lines[-1][1][-1] != '\n':
1607
# The copied annotation was from a line without a trailing EOL,
1608
# reinstate one for the content object, to ensure correct
1610
line = content._lines[-1][1] + '\n'
1611
content._lines[-1] = (content._lines[-1][0], line)
1613
if delta_seq is None:
1614
reference_content = self._get_content(parents[0], parent_texts)
1615
new_texts = content.text()
1616
old_texts = reference_content.text()
1617
delta_seq = patiencediff.PatienceSequenceMatcher(
1618
None, old_texts, new_texts)
1619
return self._make_line_delta(delta_seq, content)
1621
def _parse_record(self, version_id, data):
1622
"""Parse an original format knit record.
1624
These have the last element of the key only present in the stored data.
1626
rec, record_contents = self._parse_record_unchecked(data)
1627
self._check_header_version(rec, version_id)
1628
return record_contents, rec[3]
1630
def _parse_record_header(self, key, raw_data):
1631
"""Parse a record header for consistency.
1633
:return: the header and the decompressor stream.
1634
as (stream, header_record)
1636
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(raw_data))
1639
rec = self._check_header(key, df.readline())
1640
except Exception, e:
1641
raise KnitCorrupt(self,
1642
"While reading {%s} got %s(%s)"
1643
% (key, e.__class__.__name__, str(e)))
1646
def _parse_record_unchecked(self, data):
1648
# 4168 calls in 2880 217 internal
1649
# 4168 calls to _parse_record_header in 2121
1650
# 4168 calls to readlines in 330
1651
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(data))
1653
record_contents = df.readlines()
1654
except Exception, e:
1655
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1656
(data, e.__class__.__name__, str(e)))
1657
header = record_contents.pop(0)
1658
rec = self._split_header(header)
1659
last_line = record_contents.pop()
1660
if len(record_contents) != int(rec[2]):
1661
raise KnitCorrupt(self,
1662
'incorrect number of lines %s != %s'
1663
' for version {%s} %s'
1664
% (len(record_contents), int(rec[2]),
1665
rec[1], record_contents))
1666
if last_line != 'end %s\n' % rec[1]:
1667
raise KnitCorrupt(self,
1668
'unexpected version end line %r, wanted %r'
1669
% (last_line, rec[1]))
1671
return rec, record_contents
1673
def _read_records_iter(self, records):
1674
"""Read text records from data file and yield result.
1676
The result will be returned in whatever is the fastest to read.
1677
Not by the order requested. Also, multiple requests for the same
1678
record will only yield 1 response.
1679
:param records: A list of (key, access_memo) entries
1680
:return: Yields (key, contents, digest) in the order
1681
read, not the order requested
1686
# XXX: This smells wrong, IO may not be getting ordered right.
1687
needed_records = sorted(set(records), key=operator.itemgetter(1))
1688
if not needed_records:
1691
# The transport optimizes the fetching as well
1692
# (ie, reads continuous ranges.)
1693
raw_data = self._access.get_raw_records(
1694
[index_memo for key, index_memo in needed_records])
1696
for (key, index_memo), data in \
1697
izip(iter(needed_records), raw_data):
1698
content, digest = self._parse_record(key[-1], data)
1699
yield key, content, digest
1701
def _read_records_iter_raw(self, records):
1702
"""Read text records from data file and yield raw data.
1704
This unpacks enough of the text record to validate the id is
1705
as expected but thats all.
1707
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1709
# setup an iterator of the external records:
1710
# uses readv so nice and fast we hope.
1712
# grab the disk data needed.
1713
needed_offsets = [index_memo for key, index_memo
1715
raw_records = self._access.get_raw_records(needed_offsets)
1717
for key, index_memo in records:
1718
data = raw_records.next()
1719
# validate the header (note that we can only use the suffix in
1720
# current knit records).
1721
df, rec = self._parse_record_header(key, data)
1723
yield key, data, rec[3]
1725
def _record_to_data(self, key, digest, lines, dense_lines=None):
1726
"""Convert key, digest, lines into a raw data block.
1728
:param key: The key of the record. Currently keys are always serialised
1729
using just the trailing component.
1730
:param dense_lines: The bytes of lines but in a denser form. For
1731
instance, if lines is a list of 1000 bytestrings each ending in \n,
1732
dense_lines may be a list with one line in it, containing all the
1733
1000's lines and their \n's. Using dense_lines if it is already
1734
known is a win because the string join to create bytes in this
1735
function spends less time resizing the final string.
1736
:return: (len, a StringIO instance with the raw data ready to read.)
1738
# Note: using a string copy here increases memory pressure with e.g.
1739
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1740
# when doing the initial commit of a mozilla tree. RBC 20070921
1741
bytes = ''.join(chain(
1742
["version %s %d %s\n" % (key[-1],
1745
dense_lines or lines,
1746
["end %s\n" % key[-1]]))
1747
if type(bytes) != str:
1748
raise AssertionError(
1749
'data must be plain bytes was %s' % type(bytes))
1750
if lines and lines[-1][-1] != '\n':
1751
raise ValueError('corrupt lines value %r' % lines)
1752
compressed_bytes = tuned_gzip.bytes_to_gzip(bytes)
1753
return len(compressed_bytes), compressed_bytes
1755
def _split_header(self, line):
1758
raise KnitCorrupt(self,
1759
'unexpected number of elements in record header')
1763
"""See VersionedFiles.keys."""
1764
if 'evil' in debug.debug_flags:
1765
trace.mutter_callsite(2, "keys scales with size of history")
1766
sources = [self._index] + self._fallback_vfs
1768
for source in sources:
1769
result.update(source.keys())
1773
class _KndxIndex(object):
1774
"""Manages knit index files
1776
The index is kept in memory and read on startup, to enable
1777
fast lookups of revision information. The cursor of the index
1778
file is always pointing to the end, making it easy to append
1781
_cache is a cache for fast mapping from version id to a Index
1784
_history is a cache for fast mapping from indexes to version ids.
1786
The index data format is dictionary compressed when it comes to
1787
parent references; a index entry may only have parents that with a
1788
lover index number. As a result, the index is topological sorted.
1790
Duplicate entries may be written to the index for a single version id
1791
if this is done then the latter one completely replaces the former:
1792
this allows updates to correct version and parent information.
1793
Note that the two entries may share the delta, and that successive
1794
annotations and references MUST point to the first entry.
1796
The index file on disc contains a header, followed by one line per knit
1797
record. The same revision can be present in an index file more than once.
1798
The first occurrence gets assigned a sequence number starting from 0.
1800
The format of a single line is
1801
REVISION_ID FLAGS BYTE_OFFSET LENGTH( PARENT_ID|PARENT_SEQUENCE_ID)* :\n
1802
REVISION_ID is a utf8-encoded revision id
1803
FLAGS is a comma separated list of flags about the record. Values include
1804
no-eol, line-delta, fulltext.
1805
BYTE_OFFSET is the ascii representation of the byte offset in the data file
1806
that the the compressed data starts at.
1807
LENGTH is the ascii representation of the length of the data file.
1808
PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
1810
PARENT_SEQUENCE_ID the ascii representation of the sequence number of a
1811
revision id already in the knit that is a parent of REVISION_ID.
1812
The ' :' marker is the end of record marker.
1815
when a write is interrupted to the index file, it will result in a line
1816
that does not end in ' :'. If the ' :' is not present at the end of a line,
1817
or at the end of the file, then the record that is missing it will be
1818
ignored by the parser.
1820
When writing new records to the index file, the data is preceded by '\n'
1821
to ensure that records always start on new lines even if the last write was
1822
interrupted. As a result its normal for the last line in the index to be
1823
missing a trailing newline. One can be added with no harmful effects.
1825
:ivar _kndx_cache: dict from prefix to the old state of KnitIndex objects,
1826
where prefix is e.g. the (fileid,) for .texts instances or () for
1827
constant-mapped things like .revisions, and the old state is
1828
tuple(history_vector, cache_dict). This is used to prevent having an
1829
ABI change with the C extension that reads .kndx files.
1832
HEADER = "# bzr knit index 8\n"
1834
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1835
"""Create a _KndxIndex on transport using mapper."""
1836
self._transport = transport
1837
self._mapper = mapper
1838
self._get_scope = get_scope
1839
self._allow_writes = allow_writes
1840
self._is_locked = is_locked
1842
self.has_graph = True
1844
def add_records(self, records, random_id=False):
1845
"""Add multiple records to the index.
1847
:param records: a list of tuples:
1848
(key, options, access_memo, parents).
1849
:param random_id: If True the ids being added were randomly generated
1850
and no check for existence will be performed.
1853
for record in records:
1856
path = self._mapper.map(key) + '.kndx'
1857
path_keys = paths.setdefault(path, (prefix, []))
1858
path_keys[1].append(record)
1859
for path in sorted(paths):
1860
prefix, path_keys = paths[path]
1861
self._load_prefixes([prefix])
1863
orig_history = self._kndx_cache[prefix][1][:]
1864
orig_cache = self._kndx_cache[prefix][0].copy()
1867
for key, options, (_, pos, size), parents in path_keys:
1869
# kndx indices cannot be parentless.
1871
line = "\n%s %s %s %s %s :" % (
1872
key[-1], ','.join(options), pos, size,
1873
self._dictionary_compress(parents))
1874
if type(line) != str:
1875
raise AssertionError(
1876
'data must be utf8 was %s' % type(line))
1878
self._cache_key(key, options, pos, size, parents)
1879
if len(orig_history):
1880
self._transport.append_bytes(path, ''.join(lines))
1882
self._init_index(path, lines)
1884
# If any problems happen, restore the original values and re-raise
1885
self._kndx_cache[prefix] = (orig_cache, orig_history)
1888
def _cache_key(self, key, options, pos, size, parent_keys):
1889
"""Cache a version record in the history array and index cache.
1891
This is inlined into _load_data for performance. KEEP IN SYNC.
1892
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1896
version_id = key[-1]
1897
# last-element only for compatibilty with the C load_data.
1898
parents = tuple(parent[-1] for parent in parent_keys)
1899
for parent in parent_keys:
1900
if parent[:-1] != prefix:
1901
raise ValueError("mismatched prefixes for %r, %r" % (
1903
cache, history = self._kndx_cache[prefix]
1904
# only want the _history index to reference the 1st index entry
1906
if version_id not in cache:
1907
index = len(history)
1908
history.append(version_id)
1910
index = cache[version_id][5]
1911
cache[version_id] = (version_id,
1918
def check_header(self, fp):
1919
line = fp.readline()
1921
# An empty file can actually be treated as though the file doesn't
1923
raise errors.NoSuchFile(self)
1924
if line != self.HEADER:
1925
raise KnitHeaderError(badline=line, filename=self)
1927
def _check_read(self):
1928
if not self._is_locked():
1929
raise errors.ObjectNotLocked(self)
1930
if self._get_scope() != self._scope:
1933
def _check_write_ok(self):
1934
"""Assert if not writes are permitted."""
1935
if not self._is_locked():
1936
raise errors.ObjectNotLocked(self)
1937
if self._get_scope() != self._scope:
1939
if self._mode != 'w':
1940
raise errors.ReadOnlyObjectDirtiedError(self)
1942
def get_build_details(self, keys):
1943
"""Get the method, index_memo and compression parent for keys.
1945
Ghosts are omitted from the result.
1947
:param keys: An iterable of keys.
1948
:return: A dict of key:(index_memo, compression_parent, parents,
1951
opaque structure to pass to read_records to extract the raw
1954
Content that this record is built upon, may be None
1956
Logical parents of this node
1958
extra information about the content which needs to be passed to
1959
Factory.parse_record
1961
parent_map = self.get_parent_map(keys)
1964
if key not in parent_map:
1966
method = self.get_method(key)
1967
parents = parent_map[key]
1968
if method == 'fulltext':
1969
compression_parent = None
1971
compression_parent = parents[0]
1972
noeol = 'no-eol' in self.get_options(key)
1973
index_memo = self.get_position(key)
1974
result[key] = (index_memo, compression_parent,
1975
parents, (method, noeol))
1978
def get_method(self, key):
1979
"""Return compression method of specified key."""
1980
options = self.get_options(key)
1981
if 'fulltext' in options:
1983
elif 'line-delta' in options:
1986
raise errors.KnitIndexUnknownMethod(self, options)
1988
def get_options(self, key):
1989
"""Return a list representing options.
1993
prefix, suffix = self._split_key(key)
1994
self._load_prefixes([prefix])
1996
return self._kndx_cache[prefix][0][suffix][1]
1998
raise RevisionNotPresent(key, self)
2000
def get_parent_map(self, keys):
2001
"""Get a map of the parents of keys.
2003
:param keys: The keys to look up parents for.
2004
:return: A mapping from keys to parents. Absent keys are absent from
2007
# Parse what we need to up front, this potentially trades off I/O
2008
# locality (.kndx and .knit in the same block group for the same file
2009
# id) for less checking in inner loops.
2010
prefixes = set(key[:-1] for key in keys)
2011
self._load_prefixes(prefixes)
2016
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
2020
result[key] = tuple(prefix + (suffix,) for
2021
suffix in suffix_parents)
2024
def get_position(self, key):
2025
"""Return details needed to access the version.
2027
:return: a tuple (key, data position, size) to hand to the access
2028
logic to get the record.
2030
prefix, suffix = self._split_key(key)
2031
self._load_prefixes([prefix])
2032
entry = self._kndx_cache[prefix][0][suffix]
2033
return key, entry[2], entry[3]
2035
has_key = _mod_index._has_key_from_parent_map
2037
def _init_index(self, path, extra_lines=[]):
2038
"""Initialize an index."""
2040
sio.write(self.HEADER)
2041
sio.writelines(extra_lines)
2043
self._transport.put_file_non_atomic(path, sio,
2044
create_parent_dir=True)
2045
# self._create_parent_dir)
2046
# mode=self._file_mode,
2047
# dir_mode=self._dir_mode)
2050
"""Get all the keys in the collection.
2052
The keys are not ordered.
2055
# Identify all key prefixes.
2056
# XXX: A bit hacky, needs polish.
2057
if type(self._mapper) == ConstantMapper:
2061
for quoted_relpath in self._transport.iter_files_recursive():
2062
path, ext = os.path.splitext(quoted_relpath)
2064
prefixes = [self._mapper.unmap(path) for path in relpaths]
2065
self._load_prefixes(prefixes)
2066
for prefix in prefixes:
2067
for suffix in self._kndx_cache[prefix][1]:
2068
result.add(prefix + (suffix,))
2071
def _load_prefixes(self, prefixes):
2072
"""Load the indices for prefixes."""
2074
for prefix in prefixes:
2075
if prefix not in self._kndx_cache:
2076
# the load_data interface writes to these variables.
2079
self._filename = prefix
2081
path = self._mapper.map(prefix) + '.kndx'
2082
fp = self._transport.get(path)
2084
# _load_data may raise NoSuchFile if the target knit is
2086
_load_data(self, fp)
2089
self._kndx_cache[prefix] = (self._cache, self._history)
2094
self._kndx_cache[prefix] = ({}, [])
2095
if type(self._mapper) == ConstantMapper:
2096
# preserve behaviour for revisions.kndx etc.
2097
self._init_index(path)
2102
missing_keys = _mod_index._missing_keys_from_parent_map
2104
def _partition_keys(self, keys):
2105
"""Turn keys into a dict of prefix:suffix_list."""
2108
prefix_keys = result.setdefault(key[:-1], [])
2109
prefix_keys.append(key[-1])
2112
def _dictionary_compress(self, keys):
2113
"""Dictionary compress keys.
2115
:param keys: The keys to generate references to.
2116
:return: A string representation of keys. keys which are present are
2117
dictionary compressed, and others are emitted as fulltext with a
2123
prefix = keys[0][:-1]
2124
cache = self._kndx_cache[prefix][0]
2126
if key[:-1] != prefix:
2127
# kndx indices cannot refer across partitioned storage.
2128
raise ValueError("mismatched prefixes for %r" % keys)
2129
if key[-1] in cache:
2130
# -- inlined lookup() --
2131
result_list.append(str(cache[key[-1]][5]))
2132
# -- end lookup () --
2134
result_list.append('.' + key[-1])
2135
return ' '.join(result_list)
2137
def _reset_cache(self):
2138
# Possibly this should be a LRU cache. A dictionary from key_prefix to
2139
# (cache_dict, history_vector) for parsed kndx files.
2140
self._kndx_cache = {}
2141
self._scope = self._get_scope()
2142
allow_writes = self._allow_writes()
2148
def _sort_keys_by_io(self, keys, positions):
2149
"""Figure out an optimal order to read the records for the given keys.
2151
Sort keys, grouped by index and sorted by position.
2153
:param keys: A list of keys whose records we want to read. This will be
2155
:param positions: A dict, such as the one returned by
2156
_get_components_positions()
2159
def get_sort_key(key):
2160
index_memo = positions[key][1]
2161
# Group by prefix and position. index_memo[0] is the key, so it is
2162
# (file_id, revision_id) and we don't want to sort on revision_id,
2163
# index_memo[1] is the position, and index_memo[2] is the size,
2164
# which doesn't matter for the sort
2165
return index_memo[0][:-1], index_memo[1]
2166
return keys.sort(key=get_sort_key)
2168
def _split_key(self, key):
2169
"""Split key into a prefix and suffix."""
2170
return key[:-1], key[-1]
2173
class _KnitGraphIndex(object):
2174
"""A KnitVersionedFiles index layered on GraphIndex."""
2176
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2178
"""Construct a KnitGraphIndex on a graph_index.
2180
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2181
:param is_locked: A callback to check whether the object should answer
2183
:param deltas: Allow delta-compressed records.
2184
:param parents: If True, record knits parents, if not do not record
2186
:param add_callback: If not None, allow additions to the index and call
2187
this callback with a list of added GraphIndex nodes:
2188
[(node, value, node_refs), ...]
2189
:param is_locked: A callback, returns True if the index is locked and
2192
self._add_callback = add_callback
2193
self._graph_index = graph_index
2194
self._deltas = deltas
2195
self._parents = parents
2196
if deltas and not parents:
2197
# XXX: TODO: Delta tree and parent graph should be conceptually
2199
raise KnitCorrupt(self, "Cannot do delta compression without "
2201
self.has_graph = parents
2202
self._is_locked = is_locked
2205
return "%s(%r)" % (self.__class__.__name__, self._graph_index)
2207
def add_records(self, records, random_id=False):
2208
"""Add multiple records to the index.
2210
This function does not insert data into the Immutable GraphIndex
2211
backing the KnitGraphIndex, instead it prepares data for insertion by
2212
the caller and checks that it is safe to insert then calls
2213
self._add_callback with the prepared GraphIndex nodes.
2215
:param records: a list of tuples:
2216
(key, options, access_memo, parents).
2217
:param random_id: If True the ids being added were randomly generated
2218
and no check for existence will be performed.
2220
if not self._add_callback:
2221
raise errors.ReadOnlyError(self)
2222
# we hope there are no repositories with inconsistent parentage
2226
for (key, options, access_memo, parents) in records:
2228
parents = tuple(parents)
2229
index, pos, size = access_memo
2230
if 'no-eol' in options:
2234
value += "%d %d" % (pos, size)
2235
if not self._deltas:
2236
if 'line-delta' in options:
2237
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
2240
if 'line-delta' in options:
2241
node_refs = (parents, (parents[0],))
2243
node_refs = (parents, ())
2245
node_refs = (parents, )
2248
raise KnitCorrupt(self, "attempt to add node with parents "
2249
"in parentless index.")
2251
keys[key] = (value, node_refs)
2254
present_nodes = self._get_entries(keys)
2255
for (index, key, value, node_refs) in present_nodes:
2256
if (value[0] != keys[key][0][0] or
2257
node_refs[:1] != keys[key][1][:1]):
2258
raise KnitCorrupt(self, "inconsistent details in add_records"
2259
": %s %s" % ((value, node_refs), keys[key]))
2263
for key, (value, node_refs) in keys.iteritems():
2264
result.append((key, value, node_refs))
2266
for key, (value, node_refs) in keys.iteritems():
2267
result.append((key, value))
2268
self._add_callback(result)
2270
def _check_read(self):
2271
"""raise if reads are not permitted."""
2272
if not self._is_locked():
2273
raise errors.ObjectNotLocked(self)
2275
def _check_write_ok(self):
2276
"""Assert if writes are not permitted."""
2277
if not self._is_locked():
2278
raise errors.ObjectNotLocked(self)
2280
def _compression_parent(self, an_entry):
2281
# return the key that an_entry is compressed against, or None
2282
# Grab the second parent list (as deltas implies parents currently)
2283
compression_parents = an_entry[3][1]
2284
if not compression_parents:
2286
if len(compression_parents) != 1:
2287
raise AssertionError(
2288
"Too many compression parents: %r" % compression_parents)
2289
return compression_parents[0]
2291
def get_build_details(self, keys):
2292
"""Get the method, index_memo and compression parent for version_ids.
2294
Ghosts are omitted from the result.
2296
:param keys: An iterable of keys.
2297
:return: A dict of key:
2298
(index_memo, compression_parent, parents, record_details).
2300
opaque structure to pass to read_records to extract the raw
2303
Content that this record is built upon, may be None
2305
Logical parents of this node
2307
extra information about the content which needs to be passed to
2308
Factory.parse_record
2312
entries = self._get_entries(keys, False)
2313
for entry in entries:
2315
if not self._parents:
2318
parents = entry[3][0]
2319
if not self._deltas:
2320
compression_parent_key = None
2322
compression_parent_key = self._compression_parent(entry)
2323
noeol = (entry[2][0] == 'N')
2324
if compression_parent_key:
2325
method = 'line-delta'
2328
result[key] = (self._node_to_position(entry),
2329
compression_parent_key, parents,
2333
def _get_entries(self, keys, check_present=False):
2334
"""Get the entries for keys.
2336
:param keys: An iterable of index key tuples.
2341
for node in self._graph_index.iter_entries(keys):
2343
found_keys.add(node[1])
2345
# adapt parentless index to the rest of the code.
2346
for node in self._graph_index.iter_entries(keys):
2347
yield node[0], node[1], node[2], ()
2348
found_keys.add(node[1])
2350
missing_keys = keys.difference(found_keys)
2352
raise RevisionNotPresent(missing_keys.pop(), self)
2354
def get_method(self, key):
2355
"""Return compression method of specified key."""
2356
return self._get_method(self._get_node(key))
2358
def _get_method(self, node):
2359
if not self._deltas:
2361
if self._compression_parent(node):
2366
def _get_node(self, key):
2368
return list(self._get_entries([key]))[0]
2370
raise RevisionNotPresent(key, self)
2372
def get_options(self, key):
2373
"""Return a list representing options.
2377
node = self._get_node(key)
2378
options = [self._get_method(node)]
2379
if node[2][0] == 'N':
2380
options.append('no-eol')
2383
def get_parent_map(self, keys):
2384
"""Get a map of the parents of keys.
2386
:param keys: The keys to look up parents for.
2387
:return: A mapping from keys to parents. Absent keys are absent from
2391
nodes = self._get_entries(keys)
2395
result[node[1]] = node[3][0]
2398
result[node[1]] = None
2401
def get_position(self, key):
2402
"""Return details needed to access the version.
2404
:return: a tuple (index, data position, size) to hand to the access
2405
logic to get the record.
2407
node = self._get_node(key)
2408
return self._node_to_position(node)
2410
has_key = _mod_index._has_key_from_parent_map
2413
"""Get all the keys in the collection.
2415
The keys are not ordered.
2418
return [node[1] for node in self._graph_index.iter_all_entries()]
2420
missing_keys = _mod_index._missing_keys_from_parent_map
2422
def _node_to_position(self, node):
2423
"""Convert an index value to position details."""
2424
bits = node[2][1:].split(' ')
2425
return node[0], int(bits[0]), int(bits[1])
2427
def _sort_keys_by_io(self, keys, positions):
2428
"""Figure out an optimal order to read the records for the given keys.
2430
Sort keys, grouped by index and sorted by position.
2432
:param keys: A list of keys whose records we want to read. This will be
2434
:param positions: A dict, such as the one returned by
2435
_get_components_positions()
2438
def get_index_memo(key):
2439
# index_memo is at offset [1]. It is made up of (GraphIndex,
2440
# position, size). GI is an object, which will be unique for each
2441
# pack file. This causes us to group by pack file, then sort by
2442
# position. Size doesn't matter, but it isn't worth breaking up the
2444
return positions[key][1]
2445
return keys.sort(key=get_index_memo)
2448
class _KnitKeyAccess(object):
2449
"""Access to records in .knit files."""
2451
def __init__(self, transport, mapper):
2452
"""Create a _KnitKeyAccess with transport and mapper.
2454
:param transport: The transport the access object is rooted at.
2455
:param mapper: The mapper used to map keys to .knit files.
2457
self._transport = transport
2458
self._mapper = mapper
2460
def add_raw_records(self, key_sizes, raw_data):
2461
"""Add raw knit bytes to a storage area.
2463
The data is spooled to the container writer in one bytes-record per
2466
:param sizes: An iterable of tuples containing the key and size of each
2468
:param raw_data: A bytestring containing the data.
2469
:return: A list of memos to retrieve the record later. Each memo is an
2470
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2471
length), where the key is the record key.
2473
if type(raw_data) != str:
2474
raise AssertionError(
2475
'data must be plain bytes was %s' % type(raw_data))
2478
# TODO: This can be tuned for writing to sftp and other servers where
2479
# append() is relatively expensive by grouping the writes to each key
2481
for key, size in key_sizes:
2482
path = self._mapper.map(key)
2484
base = self._transport.append_bytes(path + '.knit',
2485
raw_data[offset:offset+size])
2486
except errors.NoSuchFile:
2487
self._transport.mkdir(osutils.dirname(path))
2488
base = self._transport.append_bytes(path + '.knit',
2489
raw_data[offset:offset+size])
2493
result.append((key, base, size))
2496
def get_raw_records(self, memos_for_retrieval):
2497
"""Get the raw bytes for a records.
2499
:param memos_for_retrieval: An iterable containing the access memo for
2500
retrieving the bytes.
2501
:return: An iterator over the bytes of the records.
2503
# first pass, group into same-index request to minimise readv's issued.
2505
current_prefix = None
2506
for (key, offset, length) in memos_for_retrieval:
2507
if current_prefix == key[:-1]:
2508
current_list.append((offset, length))
2510
if current_prefix is not None:
2511
request_lists.append((current_prefix, current_list))
2512
current_prefix = key[:-1]
2513
current_list = [(offset, length)]
2514
# handle the last entry
2515
if current_prefix is not None:
2516
request_lists.append((current_prefix, current_list))
2517
for prefix, read_vector in request_lists:
2518
path = self._mapper.map(prefix) + '.knit'
2519
for pos, data in self._transport.readv(path, read_vector):
2523
class _DirectPackAccess(object):
2524
"""Access to data in one or more packs with less translation."""
2526
def __init__(self, index_to_packs, reload_func=None):
2527
"""Create a _DirectPackAccess object.
2529
:param index_to_packs: A dict mapping index objects to the transport
2530
and file names for obtaining data.
2531
:param reload_func: A function to call if we determine that the pack
2532
files have moved and we need to reload our caches. See
2533
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
2535
self._container_writer = None
2536
self._write_index = None
2537
self._indices = index_to_packs
2538
self._reload_func = reload_func
2540
def add_raw_records(self, key_sizes, raw_data):
2541
"""Add raw knit bytes to a storage area.
2543
The data is spooled to the container writer in one bytes-record per
2546
:param sizes: An iterable of tuples containing the key and size of each
2548
:param raw_data: A bytestring containing the data.
2549
:return: A list of memos to retrieve the record later. Each memo is an
2550
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2551
length), where the index field is the write_index object supplied
2552
to the PackAccess object.
2554
if type(raw_data) != str:
2555
raise AssertionError(
2556
'data must be plain bytes was %s' % type(raw_data))
2559
for key, size in key_sizes:
2560
p_offset, p_length = self._container_writer.add_bytes_record(
2561
raw_data[offset:offset+size], [])
2563
result.append((self._write_index, p_offset, p_length))
2566
def get_raw_records(self, memos_for_retrieval):
2567
"""Get the raw bytes for a records.
2569
:param memos_for_retrieval: An iterable containing the (index, pos,
2570
length) memo for retrieving the bytes. The Pack access method
2571
looks up the pack to use for a given record in its index_to_pack
2573
:return: An iterator over the bytes of the records.
2575
# first pass, group into same-index requests
2577
current_index = None
2578
for (index, offset, length) in memos_for_retrieval:
2579
if current_index == index:
2580
current_list.append((offset, length))
2582
if current_index is not None:
2583
request_lists.append((current_index, current_list))
2584
current_index = index
2585
current_list = [(offset, length)]
2586
# handle the last entry
2587
if current_index is not None:
2588
request_lists.append((current_index, current_list))
2589
for index, offsets in request_lists:
2591
transport, path = self._indices[index]
2593
# A KeyError here indicates that someone has triggered an index
2594
# reload, and this index has gone missing, we need to start
2596
if self._reload_func is None:
2597
# If we don't have a _reload_func there is nothing that can
2600
raise errors.RetryWithNewPacks(index,
2601
reload_occurred=True,
2602
exc_info=sys.exc_info())
2604
reader = pack.make_readv_reader(transport, path, offsets)
2605
for names, read_func in reader.iter_records():
2606
yield read_func(None)
2607
except errors.NoSuchFile:
2608
# A NoSuchFile error indicates that a pack file has gone
2609
# missing on disk, we need to trigger a reload, and start over.
2610
if self._reload_func is None:
2612
raise errors.RetryWithNewPacks(transport.abspath(path),
2613
reload_occurred=False,
2614
exc_info=sys.exc_info())
2616
def set_writer(self, writer, index, transport_packname):
2617
"""Set a writer to use for adding data."""
2618
if index is not None:
2619
self._indices[index] = transport_packname
2620
self._container_writer = writer
2621
self._write_index = index
2623
def reload_or_raise(self, retry_exc):
2624
"""Try calling the reload function, or re-raise the original exception.
2626
This should be called after _DirectPackAccess raises a
2627
RetryWithNewPacks exception. This function will handle the common logic
2628
of determining when the error is fatal versus being temporary.
2629
It will also make sure that the original exception is raised, rather
2630
than the RetryWithNewPacks exception.
2632
If this function returns, then the calling function should retry
2633
whatever operation was being performed. Otherwise an exception will
2636
:param retry_exc: A RetryWithNewPacks exception.
2639
if self._reload_func is None:
2641
elif not self._reload_func():
2642
# The reload claimed that nothing changed
2643
if not retry_exc.reload_occurred:
2644
# If there wasn't an earlier reload, then we really were
2645
# expecting to find changes. We didn't find them, so this is a
2649
exc_class, exc_value, exc_traceback = retry_exc.exc_info
2650
raise exc_class, exc_value, exc_traceback
2653
# Deprecated, use PatienceSequenceMatcher instead
2654
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2657
def annotate_knit(knit, revision_id):
2658
"""Annotate a knit with no cached annotations.
2660
This implementation is for knits with no cached annotations.
2661
It will work for knits with cached annotations, but this is not
2664
annotator = _KnitAnnotator(knit)
2665
return iter(annotator.annotate(revision_id))
2668
class _KnitAnnotator(object):
2669
"""Build up the annotations for a text."""
2671
def __init__(self, knit):
2674
# Content objects, differs from fulltexts because of how final newlines
2675
# are treated by knits. the content objects here will always have a
2677
self._fulltext_contents = {}
2679
# Annotated lines of specific revisions
2680
self._annotated_lines = {}
2682
# Track the raw data for nodes that we could not process yet.
2683
# This maps the revision_id of the base to a list of children that will
2684
# annotated from it.
2685
self._pending_children = {}
2687
# Nodes which cannot be extracted
2688
self._ghosts = set()
2690
# Track how many children this node has, so we know if we need to keep
2692
self._annotate_children = {}
2693
self._compression_children = {}
2695
self._all_build_details = {}
2696
# The children => parent revision_id graph
2697
self._revision_id_graph = {}
2699
self._heads_provider = None
2701
self._nodes_to_keep_annotations = set()
2702
self._generations_until_keep = 100
2704
def set_generations_until_keep(self, value):
2705
"""Set the number of generations before caching a node.
2707
Setting this to -1 will cache every merge node, setting this higher
2708
will cache fewer nodes.
2710
self._generations_until_keep = value
2712
def _add_fulltext_content(self, revision_id, content_obj):
2713
self._fulltext_contents[revision_id] = content_obj
2714
# TODO: jam 20080305 It might be good to check the sha1digest here
2715
return content_obj.text()
2717
def _check_parents(self, child, nodes_to_annotate):
2718
"""Check if all parents have been processed.
2720
:param child: A tuple of (rev_id, parents, raw_content)
2721
:param nodes_to_annotate: If child is ready, add it to
2722
nodes_to_annotate, otherwise put it back in self._pending_children
2724
for parent_id in child[1]:
2725
if (parent_id not in self._annotated_lines):
2726
# This parent is present, but another parent is missing
2727
self._pending_children.setdefault(parent_id,
2731
# This one is ready to be processed
2732
nodes_to_annotate.append(child)
2734
def _add_annotation(self, revision_id, fulltext, parent_ids,
2735
left_matching_blocks=None):
2736
"""Add an annotation entry.
2738
All parents should already have been annotated.
2739
:return: A list of children that now have their parents satisfied.
2741
a = self._annotated_lines
2742
annotated_parent_lines = [a[p] for p in parent_ids]
2743
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2744
fulltext, revision_id, left_matching_blocks,
2745
heads_provider=self._get_heads_provider()))
2746
self._annotated_lines[revision_id] = annotated_lines
2747
for p in parent_ids:
2748
ann_children = self._annotate_children[p]
2749
ann_children.remove(revision_id)
2750
if (not ann_children
2751
and p not in self._nodes_to_keep_annotations):
2752
del self._annotated_lines[p]
2753
del self._all_build_details[p]
2754
if p in self._fulltext_contents:
2755
del self._fulltext_contents[p]
2756
# Now that we've added this one, see if there are any pending
2757
# deltas to be done, certainly this parent is finished
2758
nodes_to_annotate = []
2759
for child in self._pending_children.pop(revision_id, []):
2760
self._check_parents(child, nodes_to_annotate)
2761
return nodes_to_annotate
2763
def _get_build_graph(self, key):
2764
"""Get the graphs for building texts and annotations.
2766
The data you need for creating a full text may be different than the
2767
data you need to annotate that text. (At a minimum, you need both
2768
parents to create an annotation, but only need 1 parent to generate the
2771
:return: A list of (key, index_memo) records, suitable for
2772
passing to read_records_iter to start reading in the raw data fro/
2775
if key in self._annotated_lines:
2778
pending = set([key])
2783
# get all pending nodes
2785
this_iteration = pending
2786
build_details = self._knit._index.get_build_details(this_iteration)
2787
self._all_build_details.update(build_details)
2788
# new_nodes = self._knit._index._get_entries(this_iteration)
2790
for key, details in build_details.iteritems():
2791
(index_memo, compression_parent, parents,
2792
record_details) = details
2793
self._revision_id_graph[key] = parents
2794
records.append((key, index_memo))
2795
# Do we actually need to check _annotated_lines?
2796
pending.update(p for p in parents
2797
if p not in self._all_build_details)
2798
if compression_parent:
2799
self._compression_children.setdefault(compression_parent,
2802
for parent in parents:
2803
self._annotate_children.setdefault(parent,
2805
num_gens = generation - kept_generation
2806
if ((num_gens >= self._generations_until_keep)
2807
and len(parents) > 1):
2808
kept_generation = generation
2809
self._nodes_to_keep_annotations.add(key)
2811
missing_versions = this_iteration.difference(build_details.keys())
2812
self._ghosts.update(missing_versions)
2813
for missing_version in missing_versions:
2814
# add a key, no parents
2815
self._revision_id_graph[missing_version] = ()
2816
pending.discard(missing_version) # don't look for it
2817
if self._ghosts.intersection(self._compression_children):
2819
"We cannot have nodes which have a ghost compression parent:\n"
2821
"compression children: %r"
2822
% (self._ghosts, self._compression_children))
2823
# Cleanout anything that depends on a ghost so that we don't wait for
2824
# the ghost to show up
2825
for node in self._ghosts:
2826
if node in self._annotate_children:
2827
# We won't be building this node
2828
del self._annotate_children[node]
2829
# Generally we will want to read the records in reverse order, because
2830
# we find the parent nodes after the children
2834
def _annotate_records(self, records):
2835
"""Build the annotations for the listed records."""
2836
# We iterate in the order read, rather than a strict order requested
2837
# However, process what we can, and put off to the side things that
2838
# still need parents, cleaning them up when those parents are
2840
for (rev_id, record,
2841
digest) in self._knit._read_records_iter(records):
2842
if rev_id in self._annotated_lines:
2844
parent_ids = self._revision_id_graph[rev_id]
2845
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2846
details = self._all_build_details[rev_id]
2847
(index_memo, compression_parent, parents,
2848
record_details) = details
2849
nodes_to_annotate = []
2850
# TODO: Remove the punning between compression parents, and
2851
# parent_ids, we should be able to do this without assuming
2853
if len(parent_ids) == 0:
2854
# There are no parents for this node, so just add it
2855
# TODO: This probably needs to be decoupled
2856
fulltext_content, delta = self._knit._factory.parse_record(
2857
rev_id, record, record_details, None)
2858
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2859
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2860
parent_ids, left_matching_blocks=None))
2862
child = (rev_id, parent_ids, record)
2863
# Check if all the parents are present
2864
self._check_parents(child, nodes_to_annotate)
2865
while nodes_to_annotate:
2866
# Should we use a queue here instead of a stack?
2867
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2868
(index_memo, compression_parent, parents,
2869
record_details) = self._all_build_details[rev_id]
2871
if compression_parent is not None:
2872
comp_children = self._compression_children[compression_parent]
2873
if rev_id not in comp_children:
2874
raise AssertionError("%r not in compression children %r"
2875
% (rev_id, comp_children))
2876
# If there is only 1 child, it is safe to reuse this
2878
reuse_content = (len(comp_children) == 1
2879
and compression_parent not in
2880
self._nodes_to_keep_annotations)
2882
# Remove it from the cache since it will be changing
2883
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2884
# Make sure to copy the fulltext since it might be
2886
parent_fulltext = list(parent_fulltext_content.text())
2888
parent_fulltext_content = self._fulltext_contents[compression_parent]
2889
parent_fulltext = parent_fulltext_content.text()
2890
comp_children.remove(rev_id)
2891
fulltext_content, delta = self._knit._factory.parse_record(
2892
rev_id, record, record_details,
2893
parent_fulltext_content,
2894
copy_base_content=(not reuse_content))
2895
fulltext = self._add_fulltext_content(rev_id,
2897
if compression_parent == parent_ids[0]:
2898
# the compression_parent is the left parent, so we can
2900
blocks = KnitContent.get_line_delta_blocks(delta,
2901
parent_fulltext, fulltext)
2903
fulltext_content = self._knit._factory.parse_fulltext(
2905
fulltext = self._add_fulltext_content(rev_id,
2907
nodes_to_annotate.extend(
2908
self._add_annotation(rev_id, fulltext, parent_ids,
2909
left_matching_blocks=blocks))
2911
def _get_heads_provider(self):
2912
"""Create a heads provider for resolving ancestry issues."""
2913
if self._heads_provider is not None:
2914
return self._heads_provider
2915
parent_provider = _mod_graph.DictParentsProvider(
2916
self._revision_id_graph)
2917
graph_obj = _mod_graph.Graph(parent_provider)
2918
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2919
self._heads_provider = head_cache
2922
def annotate(self, key):
2923
"""Return the annotated fulltext at the given key.
2925
:param key: The key to annotate.
2927
if len(self._knit._fallback_vfs) > 0:
2928
# stacked knits can't use the fast path at present.
2929
return self._simple_annotate(key)
2932
records = self._get_build_graph(key)
2933
if key in self._ghosts:
2934
raise errors.RevisionNotPresent(key, self._knit)
2935
self._annotate_records(records)
2936
return self._annotated_lines[key]
2937
except errors.RetryWithNewPacks, e:
2938
self._knit._access.reload_or_raise(e)
2939
# The cached build_details are no longer valid
2940
self._all_build_details.clear()
2942
def _simple_annotate(self, key):
2943
"""Return annotated fulltext, rediffing from the full texts.
2945
This is slow but makes no assumptions about the repository
2946
being able to produce line deltas.
2948
# TODO: this code generates a parent maps of present ancestors; it
2949
# could be split out into a separate method, and probably should use
2950
# iter_ancestry instead. -- mbp and robertc 20080704
2951
graph = _mod_graph.Graph(self._knit)
2952
head_cache = _mod_graph.FrozenHeadsCache(graph)
2953
search = graph._make_breadth_first_searcher([key])
2957
present, ghosts = search.next_with_ghosts()
2958
except StopIteration:
2960
keys.update(present)
2961
parent_map = self._knit.get_parent_map(keys)
2963
reannotate = annotate.reannotate
2964
for record in self._knit.get_record_stream(keys, 'topological', True):
2966
fulltext = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
2967
parents = parent_map[key]
2968
if parents is not None:
2969
parent_lines = [parent_cache[parent] for parent in parent_map[key]]
2972
parent_cache[key] = list(
2973
reannotate(parent_lines, fulltext, key, None, head_cache))
2975
return parent_cache[key]
2977
raise errors.RevisionNotPresent(key, self._knit)
2981
from bzrlib._knit_load_data_c import _load_data_c as _load_data
2983
from bzrlib._knit_load_data_py import _load_data_py as _load_data
added
removed
bzrlib/knit.py
