1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Knit versionedfile implementation.
19
A knit is a versioned file implementation that supports efficient append only
23
lifeless: the data file is made up of "delta records". each delta record has a delta header
24
that contains; (1) a version id, (2) the size of the delta (in lines), and (3) the digest of
25
the -expanded data- (ie, the delta applied to the parent). the delta also ends with a
26
end-marker; simply "end VERSION"
28
delta can be line or full contents.a
29
... the 8's there are the index number of the annotation.
30
version robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad 7 c7d23b2a5bd6ca00e8e266cec0ec228158ee9f9e
34
8 e.set('executable', 'yes')
36
8 if elt.get('executable') == 'yes':
37
8 ie.executable = True
38
end robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad
42
09:33 < jrydberg> lifeless: each index is made up of a tuple of; version id, options, position, size, parents
43
09:33 < jrydberg> lifeless: the parents are currently dictionary compressed
44
09:33 < jrydberg> lifeless: (meaning it currently does not support ghosts)
45
09:33 < lifeless> right
46
09:33 < jrydberg> lifeless: the position and size is the range in the data file
49
so the index sequence is the dictionary compressed sequence number used
50
in the deltas to provide line annotation
55
# 10:16 < lifeless> make partial index writes safe
56
# 10:16 < lifeless> implement 'knit.check()' like weave.check()
57
# 10:17 < lifeless> record known ghosts so we can detect when they are filled in rather than the current 'reweave
59
# move sha1 out of the content so that join is faster at verifying parents
60
# record content length ?
63
from cStringIO import StringIO
64
from itertools import izip, chain
69
from bzrlib.lazy_import import lazy_import
70
lazy_import(globals(), """
90
from bzrlib.errors import (
98
RevisionAlreadyPresent,
101
from bzrlib.osutils import (
108
from bzrlib.versionedfile import (
109
AbsentContentFactory,
113
FulltextContentFactory,
119
# TODO: Split out code specific to this format into an associated object.
121
# TODO: Can we put in some kind of value to check that the index and data
122
# files belong together?
124
# TODO: accommodate binaries, perhaps by storing a byte count
126
# TODO: function to check whole file
128
# TODO: atomically append data, then measure backwards from the cursor
129
# position after writing to work out where it was located. we may need to
130
# bypass python file buffering.
132
DATA_SUFFIX = '.knit'
133
INDEX_SUFFIX = '.kndx'
136
class KnitAdapter(object):
137
"""Base class for knit record adaption."""
139
def __init__(self, basis_vf):
140
"""Create an adapter which accesses full texts from basis_vf.
142
:param basis_vf: A versioned file to access basis texts of deltas from.
143
May be None for adapters that do not need to access basis texts.
145
self._data = KnitVersionedFiles(None, None)
146
self._annotate_factory = KnitAnnotateFactory()
147
self._plain_factory = KnitPlainFactory()
148
self._basis_vf = basis_vf
151
class FTAnnotatedToUnannotated(KnitAdapter):
152
"""An adapter from FT annotated knits to unannotated ones."""
154
def get_bytes(self, factory, annotated_compressed_bytes):
156
self._data._parse_record_unchecked(annotated_compressed_bytes)
157
content = self._annotate_factory.parse_fulltext(contents, rec[1])
158
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
162
class DeltaAnnotatedToUnannotated(KnitAdapter):
163
"""An adapter for deltas from annotated to unannotated."""
165
def get_bytes(self, factory, annotated_compressed_bytes):
167
self._data._parse_record_unchecked(annotated_compressed_bytes)
168
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
170
contents = self._plain_factory.lower_line_delta(delta)
171
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
175
class FTAnnotatedToFullText(KnitAdapter):
176
"""An adapter from FT annotated knits to unannotated ones."""
178
def get_bytes(self, factory, annotated_compressed_bytes):
180
self._data._parse_record_unchecked(annotated_compressed_bytes)
181
content, delta = self._annotate_factory.parse_record(factory.key[-1],
182
contents, factory._build_details, None)
183
return ''.join(content.text())
186
class DeltaAnnotatedToFullText(KnitAdapter):
187
"""An adapter for deltas from annotated to unannotated."""
189
def get_bytes(self, factory, annotated_compressed_bytes):
191
self._data._parse_record_unchecked(annotated_compressed_bytes)
192
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
194
compression_parent = factory.parents[0]
195
basis_entry = self._basis_vf.get_record_stream(
196
[compression_parent], 'unordered', True).next()
197
if basis_entry.storage_kind == 'absent':
198
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
199
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
200
# Manually apply the delta because we have one annotated content and
202
basis_content = PlainKnitContent(basis_lines, compression_parent)
203
basis_content.apply_delta(delta, rec[1])
204
basis_content._should_strip_eol = factory._build_details[1]
205
return ''.join(basis_content.text())
208
class FTPlainToFullText(KnitAdapter):
209
"""An adapter from FT plain knits to unannotated ones."""
211
def get_bytes(self, factory, compressed_bytes):
213
self._data._parse_record_unchecked(compressed_bytes)
214
content, delta = self._plain_factory.parse_record(factory.key[-1],
215
contents, factory._build_details, None)
216
return ''.join(content.text())
219
class DeltaPlainToFullText(KnitAdapter):
220
"""An adapter for deltas from annotated to unannotated."""
222
def get_bytes(self, factory, compressed_bytes):
224
self._data._parse_record_unchecked(compressed_bytes)
225
delta = self._plain_factory.parse_line_delta(contents, rec[1])
226
compression_parent = factory.parents[0]
227
# XXX: string splitting overhead.
228
basis_entry = self._basis_vf.get_record_stream(
229
[compression_parent], 'unordered', True).next()
230
if basis_entry.storage_kind == 'absent':
231
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
232
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
233
basis_content = PlainKnitContent(basis_lines, compression_parent)
234
# Manually apply the delta because we have one annotated content and
236
content, _ = self._plain_factory.parse_record(rec[1], contents,
237
factory._build_details, basis_content)
238
return ''.join(content.text())
241
class KnitContentFactory(ContentFactory):
242
"""Content factory for streaming from knits.
244
:seealso ContentFactory:
247
def __init__(self, key, parents, build_details, sha1, raw_record,
248
annotated, knit=None):
249
"""Create a KnitContentFactory for key.
252
:param parents: The parents.
253
:param build_details: The build details as returned from
255
:param sha1: The sha1 expected from the full text of this object.
256
:param raw_record: The bytes of the knit data from disk.
257
:param annotated: True if the raw data is annotated.
259
ContentFactory.__init__(self)
262
self.parents = parents
263
if build_details[0] == 'line-delta':
268
annotated_kind = 'annotated-'
271
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
272
self._raw_record = raw_record
273
self._build_details = build_details
276
def get_bytes_as(self, storage_kind):
277
if storage_kind == self.storage_kind:
278
return self._raw_record
279
if storage_kind == 'fulltext' and self._knit is not None:
280
return self._knit.get_text(self.key[0])
282
raise errors.UnavailableRepresentation(self.key, storage_kind,
286
class KnitContent(object):
287
"""Content of a knit version to which deltas can be applied.
289
This is always stored in memory as a list of lines with \n at the end,
290
plus a flag saying if the final ending is really there or not, because that
291
corresponds to the on-disk knit representation.
295
self._should_strip_eol = False
297
def apply_delta(self, delta, new_version_id):
298
"""Apply delta to this object to become new_version_id."""
299
raise NotImplementedError(self.apply_delta)
301
def line_delta_iter(self, new_lines):
302
"""Generate line-based delta from this content to new_lines."""
303
new_texts = new_lines.text()
304
old_texts = self.text()
305
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
306
for tag, i1, i2, j1, j2 in s.get_opcodes():
309
# ofrom, oto, length, data
310
yield i1, i2, j2 - j1, new_lines._lines[j1:j2]
312
def line_delta(self, new_lines):
313
return list(self.line_delta_iter(new_lines))
316
def get_line_delta_blocks(knit_delta, source, target):
317
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
318
target_len = len(target)
321
for s_begin, s_end, t_len, new_text in knit_delta:
322
true_n = s_begin - s_pos
325
# knit deltas do not provide reliable info about whether the
326
# last line of a file matches, due to eol handling.
327
if source[s_pos + n -1] != target[t_pos + n -1]:
330
yield s_pos, t_pos, n
331
t_pos += t_len + true_n
333
n = target_len - t_pos
335
if source[s_pos + n -1] != target[t_pos + n -1]:
338
yield s_pos, t_pos, n
339
yield s_pos + (target_len - t_pos), target_len, 0
342
class AnnotatedKnitContent(KnitContent):
343
"""Annotated content."""
345
def __init__(self, lines):
346
KnitContent.__init__(self)
350
"""Return a list of (origin, text) for each content line."""
351
lines = self._lines[:]
352
if self._should_strip_eol:
353
origin, last_line = lines[-1]
354
lines[-1] = (origin, last_line.rstrip('\n'))
357
def apply_delta(self, delta, new_version_id):
358
"""Apply delta to this object to become new_version_id."""
361
for start, end, count, delta_lines in delta:
362
lines[offset+start:offset+end] = delta_lines
363
offset = offset + (start - end) + count
367
lines = [text for origin, text in self._lines]
368
except ValueError, e:
369
# most commonly (only?) caused by the internal form of the knit
370
# missing annotation information because of a bug - see thread
372
raise KnitCorrupt(self,
373
"line in annotated knit missing annotation information: %s"
375
if self._should_strip_eol:
376
lines[-1] = lines[-1].rstrip('\n')
380
return AnnotatedKnitContent(self._lines[:])
383
class PlainKnitContent(KnitContent):
384
"""Unannotated content.
386
When annotate[_iter] is called on this content, the same version is reported
387
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
391
def __init__(self, lines, version_id):
392
KnitContent.__init__(self)
394
self._version_id = version_id
397
"""Return a list of (origin, text) for each content line."""
398
return [(self._version_id, line) for line in self._lines]
400
def apply_delta(self, delta, new_version_id):
401
"""Apply delta to this object to become new_version_id."""
404
for start, end, count, delta_lines in delta:
405
lines[offset+start:offset+end] = delta_lines
406
offset = offset + (start - end) + count
407
self._version_id = new_version_id
410
return PlainKnitContent(self._lines[:], self._version_id)
414
if self._should_strip_eol:
416
lines[-1] = lines[-1].rstrip('\n')
420
class _KnitFactory(object):
421
"""Base class for common Factory functions."""
423
def parse_record(self, version_id, record, record_details,
424
base_content, copy_base_content=True):
425
"""Parse a record into a full content object.
427
:param version_id: The official version id for this content
428
:param record: The data returned by read_records_iter()
429
:param record_details: Details about the record returned by
431
:param base_content: If get_build_details returns a compression_parent,
432
you must return a base_content here, else use None
433
:param copy_base_content: When building from the base_content, decide
434
you can either copy it and return a new object, or modify it in
436
:return: (content, delta) A Content object and possibly a line-delta,
439
method, noeol = record_details
440
if method == 'line-delta':
441
if copy_base_content:
442
content = base_content.copy()
444
content = base_content
445
delta = self.parse_line_delta(record, version_id)
446
content.apply_delta(delta, version_id)
448
content = self.parse_fulltext(record, version_id)
450
content._should_strip_eol = noeol
451
return (content, delta)
454
class KnitAnnotateFactory(_KnitFactory):
455
"""Factory for creating annotated Content objects."""
459
def make(self, lines, version_id):
460
num_lines = len(lines)
461
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
463
def parse_fulltext(self, content, version_id):
464
"""Convert fulltext to internal representation
466
fulltext content is of the format
467
revid(utf8) plaintext\n
468
internal representation is of the format:
471
# TODO: jam 20070209 The tests expect this to be returned as tuples,
472
# but the code itself doesn't really depend on that.
473
# Figure out a way to not require the overhead of turning the
474
# list back into tuples.
475
lines = [tuple(line.split(' ', 1)) for line in content]
476
return AnnotatedKnitContent(lines)
478
def parse_line_delta_iter(self, lines):
479
return iter(self.parse_line_delta(lines))
481
def parse_line_delta(self, lines, version_id, plain=False):
482
"""Convert a line based delta into internal representation.
484
line delta is in the form of:
485
intstart intend intcount
487
revid(utf8) newline\n
488
internal representation is
489
(start, end, count, [1..count tuples (revid, newline)])
491
:param plain: If True, the lines are returned as a plain
492
list without annotations, not as a list of (origin, content) tuples, i.e.
493
(start, end, count, [1..count newline])
500
def cache_and_return(line):
501
origin, text = line.split(' ', 1)
502
return cache.setdefault(origin, origin), text
504
# walk through the lines parsing.
505
# Note that the plain test is explicitly pulled out of the
506
# loop to minimise any performance impact
509
start, end, count = [int(n) for n in header.split(',')]
510
contents = [next().split(' ', 1)[1] for i in xrange(count)]
511
result.append((start, end, count, contents))
514
start, end, count = [int(n) for n in header.split(',')]
515
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
516
result.append((start, end, count, contents))
519
def get_fulltext_content(self, lines):
520
"""Extract just the content lines from a fulltext."""
521
return (line.split(' ', 1)[1] for line in lines)
523
def get_linedelta_content(self, lines):
524
"""Extract just the content from a line delta.
526
This doesn't return all of the extra information stored in a delta.
527
Only the actual content lines.
532
header = header.split(',')
533
count = int(header[2])
534
for i in xrange(count):
535
origin, text = next().split(' ', 1)
538
def lower_fulltext(self, content):
539
"""convert a fulltext content record into a serializable form.
541
see parse_fulltext which this inverts.
543
# TODO: jam 20070209 We only do the caching thing to make sure that
544
# the origin is a valid utf-8 line, eventually we could remove it
545
return ['%s %s' % (o, t) for o, t in content._lines]
547
def lower_line_delta(self, delta):
548
"""convert a delta into a serializable form.
550
See parse_line_delta which this inverts.
552
# TODO: jam 20070209 We only do the caching thing to make sure that
553
# the origin is a valid utf-8 line, eventually we could remove it
555
for start, end, c, lines in delta:
556
out.append('%d,%d,%d\n' % (start, end, c))
557
out.extend(origin + ' ' + text
558
for origin, text in lines)
561
def annotate(self, knit, key):
562
content = knit._get_content(key)
563
# adjust for the fact that serialised annotations are only key suffixes
565
if type(key) == tuple:
567
origins = content.annotate()
569
for origin, line in origins:
570
result.append((prefix + (origin,), line))
573
# XXX: This smells a bit. Why would key ever be a non-tuple here?
574
# Aren't keys defined to be tuples? -- spiv 20080618
575
return content.annotate()
578
class KnitPlainFactory(_KnitFactory):
579
"""Factory for creating plain Content objects."""
583
def make(self, lines, version_id):
584
return PlainKnitContent(lines, version_id)
586
def parse_fulltext(self, content, version_id):
587
"""This parses an unannotated fulltext.
589
Note that this is not a noop - the internal representation
590
has (versionid, line) - its just a constant versionid.
592
return self.make(content, version_id)
594
def parse_line_delta_iter(self, lines, version_id):
596
num_lines = len(lines)
597
while cur < num_lines:
600
start, end, c = [int(n) for n in header.split(',')]
601
yield start, end, c, lines[cur:cur+c]
604
def parse_line_delta(self, lines, version_id):
605
return list(self.parse_line_delta_iter(lines, version_id))
607
def get_fulltext_content(self, lines):
608
"""Extract just the content lines from a fulltext."""
611
def get_linedelta_content(self, lines):
612
"""Extract just the content from a line delta.
614
This doesn't return all of the extra information stored in a delta.
615
Only the actual content lines.
620
header = header.split(',')
621
count = int(header[2])
622
for i in xrange(count):
625
def lower_fulltext(self, content):
626
return content.text()
628
def lower_line_delta(self, delta):
630
for start, end, c, lines in delta:
631
out.append('%d,%d,%d\n' % (start, end, c))
635
def annotate(self, knit, key):
636
annotator = _KnitAnnotator(knit)
637
return annotator.annotate(key)
641
def make_file_factory(annotated, mapper):
642
"""Create a factory for creating a file based KnitVersionedFiles.
644
This is only functional enough to run interface tests, it doesn't try to
645
provide a full pack environment.
647
:param annotated: knit annotations are wanted.
648
:param mapper: The mapper from keys to paths.
650
def factory(transport):
651
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
652
access = _KnitKeyAccess(transport, mapper)
653
return KnitVersionedFiles(index, access, annotated=annotated)
657
def make_pack_factory(graph, delta, keylength):
658
"""Create a factory for creating a pack based VersionedFiles.
660
This is only functional enough to run interface tests, it doesn't try to
661
provide a full pack environment.
663
:param graph: Store a graph.
664
:param delta: Delta compress contents.
665
:param keylength: How long should keys be.
667
def factory(transport):
668
parents = graph or delta
674
max_delta_chain = 200
677
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
678
key_elements=keylength)
679
stream = transport.open_write_stream('newpack')
680
writer = pack.ContainerWriter(stream.write)
682
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
683
deltas=delta, add_callback=graph_index.add_nodes)
684
access = _DirectPackAccess({})
685
access.set_writer(writer, graph_index, (transport, 'newpack'))
686
result = KnitVersionedFiles(index, access,
687
max_delta_chain=max_delta_chain)
688
result.stream = stream
689
result.writer = writer
694
def cleanup_pack_knit(versioned_files):
695
versioned_files.stream.close()
696
versioned_files.writer.end()
699
class KnitVersionedFiles(VersionedFiles):
700
"""Storage for many versioned files using knit compression.
702
Backend storage is managed by indices and data objects.
704
:ivar _index: A _KnitGraphIndex or similar that can describe the
705
parents, graph, compression and data location of entries in this
706
KnitVersionedFiles. Note that this is only the index for
707
*this* vfs; if there are fallbacks they must be queried separately.
710
def __init__(self, index, data_access, max_delta_chain=200,
711
annotated=False, reload_func=None):
712
"""Create a KnitVersionedFiles with index and data_access.
714
:param index: The index for the knit data.
715
:param data_access: The access object to store and retrieve knit
717
:param max_delta_chain: The maximum number of deltas to permit during
718
insertion. Set to 0 to prohibit the use of deltas.
719
:param annotated: Set to True to cause annotations to be calculated and
720
stored during insertion.
721
:param reload_func: An function that can be called if we think we need
722
to reload the pack listing and try again. See
723
'bzrlib.repofmt.pack_repo.AggregateIndex' for the signature.
726
self._access = data_access
727
self._max_delta_chain = max_delta_chain
729
self._factory = KnitAnnotateFactory()
731
self._factory = KnitPlainFactory()
732
self._fallback_vfs = []
733
self._reload_func = reload_func
736
return "%s(%r, %r)" % (
737
self.__class__.__name__,
741
def add_fallback_versioned_files(self, a_versioned_files):
742
"""Add a source of texts for texts not present in this knit.
744
:param a_versioned_files: A VersionedFiles object.
746
self._fallback_vfs.append(a_versioned_files)
748
def add_lines(self, key, parents, lines, parent_texts=None,
749
left_matching_blocks=None, nostore_sha=None, random_id=False,
751
"""See VersionedFiles.add_lines()."""
752
self._index._check_write_ok()
753
self._check_add(key, lines, random_id, check_content)
755
# The caller might pass None if there is no graph data, but kndx
756
# indexes can't directly store that, so we give them
757
# an empty tuple instead.
759
return self._add(key, lines, parents,
760
parent_texts, left_matching_blocks, nostore_sha, random_id)
762
def _add(self, key, lines, parents, parent_texts,
763
left_matching_blocks, nostore_sha, random_id):
764
"""Add a set of lines on top of version specified by parents.
766
Any versions not present will be converted into ghosts.
768
# first thing, if the content is something we don't need to store, find
770
line_bytes = ''.join(lines)
771
digest = sha_string(line_bytes)
772
if nostore_sha == digest:
773
raise errors.ExistingContent
776
if parent_texts is None:
778
# Do a single query to ascertain parent presence; we only compress
779
# against parents in the same kvf.
780
present_parent_map = self._index.get_parent_map(parents)
781
for parent in parents:
782
if parent in present_parent_map:
783
present_parents.append(parent)
785
# Currently we can only compress against the left most present parent.
786
if (len(present_parents) == 0 or
787
present_parents[0] != parents[0]):
790
# To speed the extract of texts the delta chain is limited
791
# to a fixed number of deltas. This should minimize both
792
# I/O and the time spend applying deltas.
793
delta = self._check_should_delta(present_parents[0])
795
text_length = len(line_bytes)
798
if lines[-1][-1] != '\n':
799
# copy the contents of lines.
801
options.append('no-eol')
802
lines[-1] = lines[-1] + '\n'
806
if type(element) != str:
807
raise TypeError("key contains non-strings: %r" % (key,))
808
# Knit hunks are still last-element only
810
content = self._factory.make(lines, version_id)
811
if 'no-eol' in options:
812
# Hint to the content object that its text() call should strip the
814
content._should_strip_eol = True
815
if delta or (self._factory.annotated and len(present_parents) > 0):
816
# Merge annotations from parent texts if needed.
817
delta_hunks = self._merge_annotations(content, present_parents,
818
parent_texts, delta, self._factory.annotated,
819
left_matching_blocks)
822
options.append('line-delta')
823
store_lines = self._factory.lower_line_delta(delta_hunks)
824
size, bytes = self._record_to_data(key, digest,
827
options.append('fulltext')
828
# isinstance is slower and we have no hierarchy.
829
if self._factory.__class__ == KnitPlainFactory:
830
# Use the already joined bytes saving iteration time in
832
size, bytes = self._record_to_data(key, digest,
835
# get mixed annotation + content and feed it into the
837
store_lines = self._factory.lower_fulltext(content)
838
size, bytes = self._record_to_data(key, digest,
841
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
842
self._index.add_records(
843
((key, options, access_memo, parents),),
845
return digest, text_length, content
847
def annotate(self, key):
848
"""See VersionedFiles.annotate."""
849
return self._factory.annotate(self, key)
851
def check(self, progress_bar=None):
852
"""See VersionedFiles.check()."""
853
# This doesn't actually test extraction of everything, but that will
854
# impact 'bzr check' substantially, and needs to be integrated with
855
# care. However, it does check for the obvious problem of a delta with
857
keys = self._index.keys()
858
parent_map = self.get_parent_map(keys)
860
if self._index.get_method(key) != 'fulltext':
861
compression_parent = parent_map[key][0]
862
if compression_parent not in parent_map:
863
raise errors.KnitCorrupt(self,
864
"Missing basis parent %s for %s" % (
865
compression_parent, key))
866
for fallback_vfs in self._fallback_vfs:
869
def _check_add(self, key, lines, random_id, check_content):
870
"""check that version_id and lines are safe to add."""
872
if contains_whitespace(version_id):
873
raise InvalidRevisionId(version_id, self)
874
self.check_not_reserved_id(version_id)
875
# TODO: If random_id==False and the key is already present, we should
876
# probably check that the existing content is identical to what is
877
# being inserted, and otherwise raise an exception. This would make
878
# the bundle code simpler.
880
self._check_lines_not_unicode(lines)
881
self._check_lines_are_lines(lines)
883
def _check_header(self, key, line):
884
rec = self._split_header(line)
885
self._check_header_version(rec, key[-1])
888
def _check_header_version(self, rec, version_id):
889
"""Checks the header version on original format knit records.
891
These have the last component of the key embedded in the record.
893
if rec[1] != version_id:
894
raise KnitCorrupt(self,
895
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
897
def _check_should_delta(self, parent):
898
"""Iterate back through the parent listing, looking for a fulltext.
900
This is used when we want to decide whether to add a delta or a new
901
fulltext. It searches for _max_delta_chain parents. When it finds a
902
fulltext parent, it sees if the total size of the deltas leading up to
903
it is large enough to indicate that we want a new full text anyway.
905
Return True if we should create a new delta, False if we should use a
910
for count in xrange(self._max_delta_chain):
911
# XXX: Collapse these two queries:
913
# Note that this only looks in the index of this particular
914
# KnitVersionedFiles, not in the fallbacks. This ensures that
915
# we won't store a delta spanning physical repository
917
method = self._index.get_method(parent)
918
except RevisionNotPresent:
919
# Some basis is not locally present: always delta
921
index, pos, size = self._index.get_position(parent)
922
if method == 'fulltext':
926
# We don't explicitly check for presence because this is in an
927
# inner loop, and if it's missing it'll fail anyhow.
928
# TODO: This should be asking for compression parent, not graph
930
parent = self._index.get_parent_map([parent])[parent][0]
932
# We couldn't find a fulltext, so we must create a new one
934
# Simple heuristic - if the total I/O wold be greater as a delta than
935
# the originally installed fulltext, we create a new fulltext.
936
return fulltext_size > delta_size
938
def _build_details_to_components(self, build_details):
939
"""Convert a build_details tuple to a position tuple."""
940
# record_details, access_memo, compression_parent
941
return build_details[3], build_details[0], build_details[1]
943
def _get_components_positions(self, keys, allow_missing=False):
944
"""Produce a map of position data for the components of keys.
946
This data is intended to be used for retrieving the knit records.
948
A dict of key to (record_details, index_memo, next, parents) is
950
method is the way referenced data should be applied.
951
index_memo is the handle to pass to the data access to actually get the
953
next is the build-parent of the version, or None for fulltexts.
954
parents is the version_ids of the parents of this version
956
:param allow_missing: If True do not raise an error on a missing component,
960
pending_components = keys
961
while pending_components:
962
build_details = self._index.get_build_details(pending_components)
963
current_components = set(pending_components)
964
pending_components = set()
965
for key, details in build_details.iteritems():
966
(index_memo, compression_parent, parents,
967
record_details) = details
968
method = record_details[0]
969
if compression_parent is not None:
970
pending_components.add(compression_parent)
971
component_data[key] = self._build_details_to_components(details)
972
missing = current_components.difference(build_details)
973
if missing and not allow_missing:
974
raise errors.RevisionNotPresent(missing.pop(), self)
975
return component_data
977
def _get_content(self, key, parent_texts={}):
978
"""Returns a content object that makes up the specified
980
cached_version = parent_texts.get(key, None)
981
if cached_version is not None:
982
# Ensure the cache dict is valid.
983
if not self.get_parent_map([key]):
984
raise RevisionNotPresent(key, self)
985
return cached_version
986
text_map, contents_map = self._get_content_maps([key])
987
return contents_map[key]
989
def _get_content_maps(self, keys, nonlocal_keys=None):
990
"""Produce maps of text and KnitContents
992
:param keys: The keys to produce content maps for.
993
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
994
which are known to not be in this knit, but rather in one of the
996
:return: (text_map, content_map) where text_map contains the texts for
997
the requested versions and content_map contains the KnitContents.
999
# FUTURE: This function could be improved for the 'extract many' case
1000
# by tracking each component and only doing the copy when the number of
1001
# children than need to apply delta's to it is > 1 or it is part of the
1004
multiple_versions = len(keys) != 1
1005
record_map = self._get_record_map(keys, allow_missing=True)
1010
if nonlocal_keys is None:
1011
nonlocal_keys = set()
1013
nonlocal_keys = frozenset(nonlocal_keys)
1014
missing_keys = set(nonlocal_keys)
1015
for source in self._fallback_vfs:
1016
if not missing_keys:
1018
for record in source.get_record_stream(missing_keys,
1020
if record.storage_kind == 'absent':
1022
missing_keys.remove(record.key)
1023
lines = split_lines(record.get_bytes_as('fulltext'))
1024
text_map[record.key] = lines
1025
content_map[record.key] = PlainKnitContent(lines, record.key)
1026
if record.key in keys:
1027
final_content[record.key] = content_map[record.key]
1029
if key in nonlocal_keys:
1034
while cursor is not None:
1036
record, record_details, digest, next = record_map[cursor]
1038
raise RevisionNotPresent(cursor, self)
1039
components.append((cursor, record, record_details, digest))
1041
if cursor in content_map:
1042
# no need to plan further back
1043
components.append((cursor, None, None, None))
1047
for (component_id, record, record_details,
1048
digest) in reversed(components):
1049
if component_id in content_map:
1050
content = content_map[component_id]
1052
content, delta = self._factory.parse_record(key[-1],
1053
record, record_details, content,
1054
copy_base_content=multiple_versions)
1055
if multiple_versions:
1056
content_map[component_id] = content
1058
final_content[key] = content
1060
# digest here is the digest from the last applied component.
1061
text = content.text()
1062
actual_sha = sha_strings(text)
1063
if actual_sha != digest:
1064
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
1065
text_map[key] = text
1066
return text_map, final_content
1068
def get_parent_map(self, keys):
1069
"""Get a map of the graph parents of keys.
1071
:param keys: The keys to look up parents for.
1072
:return: A mapping from keys to parents. Absent keys are absent from
1075
return self._get_parent_map_with_sources(keys)[0]
1077
def _get_parent_map_with_sources(self, keys):
1078
"""Get a map of the parents of keys.
1080
:param keys: The keys to look up parents for.
1081
:return: A tuple. The first element is a mapping from keys to parents.
1082
Absent keys are absent from the mapping. The second element is a
1083
list with the locations each key was found in. The first element
1084
is the in-this-knit parents, the second the first fallback source,
1088
sources = [self._index] + self._fallback_vfs
1091
for source in sources:
1094
new_result = source.get_parent_map(missing)
1095
source_results.append(new_result)
1096
result.update(new_result)
1097
missing.difference_update(set(new_result))
1098
return result, source_results
1100
def _get_record_map(self, keys, allow_missing=False):
1101
"""Produce a dictionary of knit records.
1103
:return: {key:(record, record_details, digest, next)}
1105
data returned from read_records
1107
opaque information to pass to parse_record
1109
SHA1 digest of the full text after all steps are done
1111
build-parent of the version, i.e. the leftmost ancestor.
1112
Will be None if the record is not a delta.
1113
:param keys: The keys to build a map for
1114
:param allow_missing: If some records are missing, rather than
1115
error, just return the data that could be generated.
1117
# This retries the whole request if anything fails. Potentially we
1118
# could be a bit more selective. We could track the keys whose records
1119
# we have successfully found, and then only request the new records
1120
# from there. However, _get_components_positions grabs the whole build
1121
# chain, which means we'll likely try to grab the same records again
1122
# anyway. Also, can the build chains change as part of a pack
1123
# operation? We wouldn't want to end up with a broken chain.
1126
position_map = self._get_components_positions(keys,
1127
allow_missing=allow_missing)
1128
# key = component_id, r = record_details, i_m = index_memo,
1130
records = [(key, i_m) for key, (r, i_m, n)
1131
in position_map.iteritems()]
1133
for key, record, digest in self._read_records_iter(records):
1134
(record_details, index_memo, next) = position_map[key]
1135
record_map[key] = record, record_details, digest, next
1137
except errors.RetryWithNewPacks, e:
1138
self._access.reload_or_raise(e)
1140
def _split_by_prefix(self, keys):
1141
"""For the given keys, split them up based on their prefix.
1143
To keep memory pressure somewhat under control, split the
1144
requests back into per-file-id requests, otherwise "bzr co"
1145
extracts the full tree into memory before writing it to disk.
1146
This should be revisited if _get_content_maps() can ever cross
1149
:param keys: An iterable of key tuples
1150
:return: A dict of {prefix: [key_list]}
1152
split_by_prefix = {}
1155
split_by_prefix.setdefault('', []).append(key)
1157
split_by_prefix.setdefault(key[0], []).append(key)
1158
return split_by_prefix
1160
def get_record_stream(self, keys, ordering, include_delta_closure):
1161
"""Get a stream of records for keys.
1163
:param keys: The keys to include.
1164
:param ordering: Either 'unordered' or 'topological'. A topologically
1165
sorted stream has compression parents strictly before their
1167
:param include_delta_closure: If True then the closure across any
1168
compression parents will be included (in the opaque data).
1169
:return: An iterator of ContentFactory objects, each of which is only
1170
valid until the iterator is advanced.
1172
# keys might be a generator
1176
if not self._index.has_graph:
1177
# Cannot topological order when no graph has been stored.
1178
ordering = 'unordered'
1180
remaining_keys = keys
1183
keys = set(remaining_keys)
1184
for content_factory in self._get_remaining_record_stream(keys,
1185
ordering, include_delta_closure):
1186
remaining_keys.discard(content_factory.key)
1187
yield content_factory
1189
except errors.RetryWithNewPacks, e:
1190
self._access.reload_or_raise(e)
1192
def _get_remaining_record_stream(self, keys, ordering,
1193
include_delta_closure):
1194
"""This function is the 'retry' portion for get_record_stream."""
1195
if include_delta_closure:
1196
positions = self._get_components_positions(keys, allow_missing=True)
1198
build_details = self._index.get_build_details(keys)
1200
# (record_details, access_memo, compression_parent_key)
1201
positions = dict((key, self._build_details_to_components(details))
1202
for key, details in build_details.iteritems())
1203
absent_keys = keys.difference(set(positions))
1204
# There may be more absent keys : if we're missing the basis component
1205
# and are trying to include the delta closure.
1206
if include_delta_closure:
1207
needed_from_fallback = set()
1208
# Build up reconstructable_keys dict. key:True in this dict means
1209
# the key can be reconstructed.
1210
reconstructable_keys = {}
1214
chain = [key, positions[key][2]]
1216
needed_from_fallback.add(key)
1219
while chain[-1] is not None:
1220
if chain[-1] in reconstructable_keys:
1221
result = reconstructable_keys[chain[-1]]
1225
chain.append(positions[chain[-1]][2])
1227
# missing basis component
1228
needed_from_fallback.add(chain[-1])
1231
for chain_key in chain[:-1]:
1232
reconstructable_keys[chain_key] = result
1234
needed_from_fallback.add(key)
1235
# Double index lookups here : need a unified api ?
1236
global_map, parent_maps = self._get_parent_map_with_sources(keys)
1237
if ordering == 'topological':
1238
# Global topological sort
1239
present_keys = tsort.topo_sort(global_map)
1240
# Now group by source:
1242
current_source = None
1243
for key in present_keys:
1244
for parent_map in parent_maps:
1245
if key in parent_map:
1246
key_source = parent_map
1248
if current_source is not key_source:
1249
source_keys.append((key_source, []))
1250
current_source = key_source
1251
source_keys[-1][1].append(key)
1253
if ordering != 'unordered':
1254
raise AssertionError('valid values for ordering are:'
1255
' "unordered" or "topological" not: %r'
1257
# Just group by source; remote sources first.
1260
for parent_map in reversed(parent_maps):
1261
source_keys.append((parent_map, []))
1262
for key in parent_map:
1263
present_keys.append(key)
1264
source_keys[-1][1].append(key)
1265
# We have been requested to return these records in an order that
1266
# suits us. So we ask the index to give us an optimally sorted
1268
for source, sub_keys in source_keys:
1269
if source is parent_maps[0]:
1270
# Only sort the keys for this VF
1271
self._index._sort_keys_by_io(sub_keys, positions)
1272
absent_keys = keys - set(global_map)
1273
for key in absent_keys:
1274
yield AbsentContentFactory(key)
1275
# restrict our view to the keys we can answer.
1276
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1277
# XXX: At that point we need to consider the impact of double reads by
1278
# utilising components multiple times.
1279
if include_delta_closure:
1280
# XXX: get_content_maps performs its own index queries; allow state
1282
non_local_keys = needed_from_fallback - absent_keys
1283
prefix_split_keys = self._split_by_prefix(present_keys)
1284
prefix_split_non_local_keys = self._split_by_prefix(non_local_keys)
1285
for prefix, keys in prefix_split_keys.iteritems():
1286
non_local = prefix_split_non_local_keys.get(prefix, [])
1287
non_local = set(non_local)
1288
text_map, _ = self._get_content_maps(keys, non_local)
1290
lines = text_map.pop(key)
1291
text = ''.join(lines)
1292
yield FulltextContentFactory(key, global_map[key], None,
1295
for source, keys in source_keys:
1296
if source is parent_maps[0]:
1297
# this KnitVersionedFiles
1298
records = [(key, positions[key][1]) for key in keys]
1299
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1300
(record_details, index_memo, _) = positions[key]
1301
yield KnitContentFactory(key, global_map[key],
1302
record_details, sha1, raw_data, self._factory.annotated, None)
1304
vf = self._fallback_vfs[parent_maps.index(source) - 1]
1305
for record in vf.get_record_stream(keys, ordering,
1306
include_delta_closure):
1309
def get_sha1s(self, keys):
1310
"""See VersionedFiles.get_sha1s()."""
1312
record_map = self._get_record_map(missing, allow_missing=True)
1314
for key, details in record_map.iteritems():
1315
if key not in missing:
1317
# record entry 2 is the 'digest'.
1318
result[key] = details[2]
1319
missing.difference_update(set(result))
1320
for source in self._fallback_vfs:
1323
new_result = source.get_sha1s(missing)
1324
result.update(new_result)
1325
missing.difference_update(set(new_result))
1328
def insert_record_stream(self, stream):
1329
"""Insert a record stream into this container.
1331
:param stream: A stream of records to insert.
1333
:seealso VersionedFiles.get_record_stream:
1335
def get_adapter(adapter_key):
1337
return adapters[adapter_key]
1339
adapter_factory = adapter_registry.get(adapter_key)
1340
adapter = adapter_factory(self)
1341
adapters[adapter_key] = adapter
1344
if self._factory.annotated:
1345
# self is annotated, we need annotated knits to use directly.
1346
annotated = "annotated-"
1349
# self is not annotated, but we can strip annotations cheaply.
1351
convertibles = set(["knit-annotated-ft-gz"])
1352
if self._max_delta_chain:
1353
delta_types.add("knit-annotated-delta-gz")
1354
convertibles.add("knit-annotated-delta-gz")
1355
# The set of types we can cheaply adapt without needing basis texts.
1356
native_types = set()
1357
if self._max_delta_chain:
1358
native_types.add("knit-%sdelta-gz" % annotated)
1359
delta_types.add("knit-%sdelta-gz" % annotated)
1360
native_types.add("knit-%sft-gz" % annotated)
1361
knit_types = native_types.union(convertibles)
1363
# Buffer all index entries that we can't add immediately because their
1364
# basis parent is missing. We don't buffer all because generating
1365
# annotations may require access to some of the new records. However we
1366
# can't generate annotations from new deltas until their basis parent
1367
# is present anyway, so we get away with not needing an index that
1368
# includes the new keys.
1370
# See <http://launchpad.net/bugs/300177> about ordering of compression
1371
# parents in the records - to be conservative, we insist that all
1372
# parents must be present to avoid expanding to a fulltext.
1374
# key = basis_parent, value = index entry to add
1375
buffered_index_entries = {}
1376
for record in stream:
1377
parents = record.parents
1378
if record.storage_kind in delta_types:
1379
# TODO: eventually the record itself should track
1380
# compression_parent
1381
compression_parent = parents[0]
1383
compression_parent = None
1384
# Raise an error when a record is missing.
1385
if record.storage_kind == 'absent':
1386
raise RevisionNotPresent([record.key], self)
1387
elif ((record.storage_kind in knit_types)
1388
and (compression_parent is None
1389
or not self._fallback_vfs
1390
or self._index.has_key(compression_parent)
1391
or not self.has_key(compression_parent))):
1392
# we can insert the knit record literally if either it has no
1393
# compression parent OR we already have its basis in this kvf
1394
# OR the basis is not present even in the fallbacks. In the
1395
# last case it will either turn up later in the stream and all
1396
# will be well, or it won't turn up at all and we'll raise an
1399
# TODO: self.has_key is somewhat redundant with
1400
# self._index.has_key; we really want something that directly
1401
# asks if it's only present in the fallbacks. -- mbp 20081119
1402
if record.storage_kind not in native_types:
1404
adapter_key = (record.storage_kind, "knit-delta-gz")
1405
adapter = get_adapter(adapter_key)
1407
adapter_key = (record.storage_kind, "knit-ft-gz")
1408
adapter = get_adapter(adapter_key)
1409
bytes = adapter.get_bytes(
1410
record, record.get_bytes_as(record.storage_kind))
1412
bytes = record.get_bytes_as(record.storage_kind)
1413
options = [record._build_details[0]]
1414
if record._build_details[1]:
1415
options.append('no-eol')
1416
# Just blat it across.
1417
# Note: This does end up adding data on duplicate keys. As
1418
# modern repositories use atomic insertions this should not
1419
# lead to excessive growth in the event of interrupted fetches.
1420
# 'knit' repositories may suffer excessive growth, but as a
1421
# deprecated format this is tolerable. It can be fixed if
1422
# needed by in the kndx index support raising on a duplicate
1423
# add with identical parents and options.
1424
access_memo = self._access.add_raw_records(
1425
[(record.key, len(bytes))], bytes)[0]
1426
index_entry = (record.key, options, access_memo, parents)
1428
if 'fulltext' not in options:
1429
# Not a fulltext, so we need to make sure the compression
1430
# parent will also be present.
1431
# Note that pack backed knits don't need to buffer here
1432
# because they buffer all writes to the transaction level,
1433
# but we don't expose that difference at the index level. If
1434
# the query here has sufficient cost to show up in
1435
# profiling we should do that.
1437
# They're required to be physically in this
1438
# KnitVersionedFiles, not in a fallback.
1439
if not self._index.has_key(compression_parent):
1440
pending = buffered_index_entries.setdefault(
1441
compression_parent, [])
1442
pending.append(index_entry)
1445
self._index.add_records([index_entry])
1446
elif record.storage_kind == 'fulltext':
1447
self.add_lines(record.key, parents,
1448
split_lines(record.get_bytes_as('fulltext')))
1450
# Not a fulltext, and not suitable for direct insertion as a
1451
# delta, either because it's not the right format, or this
1452
# KnitVersionedFiles doesn't permit deltas (_max_delta_chain ==
1453
# 0) or because it depends on a base only present in the
1455
adapter_key = record.storage_kind, 'fulltext'
1456
adapter = get_adapter(adapter_key)
1457
lines = split_lines(adapter.get_bytes(
1458
record, record.get_bytes_as(record.storage_kind)))
1460
self.add_lines(record.key, parents, lines)
1461
except errors.RevisionAlreadyPresent:
1463
# Add any records whose basis parent is now available.
1464
added_keys = [record.key]
1466
key = added_keys.pop(0)
1467
if key in buffered_index_entries:
1468
index_entries = buffered_index_entries[key]
1469
self._index.add_records(index_entries)
1471
[index_entry[0] for index_entry in index_entries])
1472
del buffered_index_entries[key]
1473
# If there were any deltas which had a missing basis parent, error.
1474
if buffered_index_entries:
1475
from pprint import pformat
1476
raise errors.BzrCheckError(
1477
"record_stream refers to compression parents not in %r:\n%s"
1478
% (self, pformat(sorted(buffered_index_entries.keys()))))
1480
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1481
"""Iterate over the lines in the versioned files from keys.
1483
This may return lines from other keys. Each item the returned
1484
iterator yields is a tuple of a line and a text version that that line
1485
is present in (not introduced in).
1487
Ordering of results is in whatever order is most suitable for the
1488
underlying storage format.
1490
If a progress bar is supplied, it may be used to indicate progress.
1491
The caller is responsible for cleaning up progress bars (because this
1495
* Lines are normalised by the underlying store: they will all have \\n
1497
* Lines are returned in arbitrary order.
1498
* If a requested key did not change any lines (or didn't have any
1499
lines), it may not be mentioned at all in the result.
1501
:return: An iterator over (line, key).
1504
pb = progress.DummyProgress()
1510
# we don't care about inclusions, the caller cares.
1511
# but we need to setup a list of records to visit.
1512
# we need key, position, length
1514
build_details = self._index.get_build_details(keys)
1515
for key, details in build_details.iteritems():
1517
key_records.append((key, details[0]))
1518
records_iter = enumerate(self._read_records_iter(key_records))
1519
for (key_idx, (key, data, sha_value)) in records_iter:
1520
pb.update('Walking content.', key_idx, total)
1521
compression_parent = build_details[key][1]
1522
if compression_parent is None:
1524
line_iterator = self._factory.get_fulltext_content(data)
1527
line_iterator = self._factory.get_linedelta_content(data)
1528
# Now that we are yielding the data for this key, remove it
1531
# XXX: It might be more efficient to yield (key,
1532
# line_iterator) in the future. However for now, this is a
1533
# simpler change to integrate into the rest of the
1534
# codebase. RBC 20071110
1535
for line in line_iterator:
1538
except errors.RetryWithNewPacks, e:
1539
self._access.reload_or_raise(e)
1540
# If there are still keys we've not yet found, we look in the fallback
1541
# vfs, and hope to find them there. Note that if the keys are found
1542
# but had no changes or no content, the fallback may not return
1544
if keys and not self._fallback_vfs:
1545
# XXX: strictly the second parameter is meant to be the file id
1546
# but it's not easily accessible here.
1547
raise RevisionNotPresent(keys, repr(self))
1548
for source in self._fallback_vfs:
1552
for line, key in source.iter_lines_added_or_present_in_keys(keys):
1553
source_keys.add(key)
1555
keys.difference_update(source_keys)
1556
pb.update('Walking content.', total, total)
1558
def _make_line_delta(self, delta_seq, new_content):
1559
"""Generate a line delta from delta_seq and new_content."""
1561
for op in delta_seq.get_opcodes():
1562
if op[0] == 'equal':
1564
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1567
def _merge_annotations(self, content, parents, parent_texts={},
1568
delta=None, annotated=None,
1569
left_matching_blocks=None):
1570
"""Merge annotations for content and generate deltas.
1572
This is done by comparing the annotations based on changes to the text
1573
and generating a delta on the resulting full texts. If annotations are
1574
not being created then a simple delta is created.
1576
if left_matching_blocks is not None:
1577
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1581
for parent_key in parents:
1582
merge_content = self._get_content(parent_key, parent_texts)
1583
if (parent_key == parents[0] and delta_seq is not None):
1586
seq = patiencediff.PatienceSequenceMatcher(
1587
None, merge_content.text(), content.text())
1588
for i, j, n in seq.get_matching_blocks():
1591
# this copies (origin, text) pairs across to the new
1592
# content for any line that matches the last-checked
1594
content._lines[j:j+n] = merge_content._lines[i:i+n]
1595
# XXX: Robert says the following block is a workaround for a
1596
# now-fixed bug and it can probably be deleted. -- mbp 20080618
1597
if content._lines and content._lines[-1][1][-1] != '\n':
1598
# The copied annotation was from a line without a trailing EOL,
1599
# reinstate one for the content object, to ensure correct
1601
line = content._lines[-1][1] + '\n'
1602
content._lines[-1] = (content._lines[-1][0], line)
1604
if delta_seq is None:
1605
reference_content = self._get_content(parents[0], parent_texts)
1606
new_texts = content.text()
1607
old_texts = reference_content.text()
1608
delta_seq = patiencediff.PatienceSequenceMatcher(
1609
None, old_texts, new_texts)
1610
return self._make_line_delta(delta_seq, content)
1612
def _parse_record(self, version_id, data):
1613
"""Parse an original format knit record.
1615
These have the last element of the key only present in the stored data.
1617
rec, record_contents = self._parse_record_unchecked(data)
1618
self._check_header_version(rec, version_id)
1619
return record_contents, rec[3]
1621
def _parse_record_header(self, key, raw_data):
1622
"""Parse a record header for consistency.
1624
:return: the header and the decompressor stream.
1625
as (stream, header_record)
1627
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(raw_data))
1630
rec = self._check_header(key, df.readline())
1631
except Exception, e:
1632
raise KnitCorrupt(self,
1633
"While reading {%s} got %s(%s)"
1634
% (key, e.__class__.__name__, str(e)))
1637
def _parse_record_unchecked(self, data):
1639
# 4168 calls in 2880 217 internal
1640
# 4168 calls to _parse_record_header in 2121
1641
# 4168 calls to readlines in 330
1642
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(data))
1644
record_contents = df.readlines()
1645
except Exception, e:
1646
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1647
(data, e.__class__.__name__, str(e)))
1648
header = record_contents.pop(0)
1649
rec = self._split_header(header)
1650
last_line = record_contents.pop()
1651
if len(record_contents) != int(rec[2]):
1652
raise KnitCorrupt(self,
1653
'incorrect number of lines %s != %s'
1654
' for version {%s} %s'
1655
% (len(record_contents), int(rec[2]),
1656
rec[1], record_contents))
1657
if last_line != 'end %s\n' % rec[1]:
1658
raise KnitCorrupt(self,
1659
'unexpected version end line %r, wanted %r'
1660
% (last_line, rec[1]))
1662
return rec, record_contents
1664
def _read_records_iter(self, records):
1665
"""Read text records from data file and yield result.
1667
The result will be returned in whatever is the fastest to read.
1668
Not by the order requested. Also, multiple requests for the same
1669
record will only yield 1 response.
1670
:param records: A list of (key, access_memo) entries
1671
:return: Yields (key, contents, digest) in the order
1672
read, not the order requested
1677
# XXX: This smells wrong, IO may not be getting ordered right.
1678
needed_records = sorted(set(records), key=operator.itemgetter(1))
1679
if not needed_records:
1682
# The transport optimizes the fetching as well
1683
# (ie, reads continuous ranges.)
1684
raw_data = self._access.get_raw_records(
1685
[index_memo for key, index_memo in needed_records])
1687
for (key, index_memo), data in \
1688
izip(iter(needed_records), raw_data):
1689
content, digest = self._parse_record(key[-1], data)
1690
yield key, content, digest
1692
def _read_records_iter_raw(self, records):
1693
"""Read text records from data file and yield raw data.
1695
This unpacks enough of the text record to validate the id is
1696
as expected but thats all.
1698
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1700
# setup an iterator of the external records:
1701
# uses readv so nice and fast we hope.
1703
# grab the disk data needed.
1704
needed_offsets = [index_memo for key, index_memo
1706
raw_records = self._access.get_raw_records(needed_offsets)
1708
for key, index_memo in records:
1709
data = raw_records.next()
1710
# validate the header (note that we can only use the suffix in
1711
# current knit records).
1712
df, rec = self._parse_record_header(key, data)
1714
yield key, data, rec[3]
1716
def _record_to_data(self, key, digest, lines, dense_lines=None):
1717
"""Convert key, digest, lines into a raw data block.
1719
:param key: The key of the record. Currently keys are always serialised
1720
using just the trailing component.
1721
:param dense_lines: The bytes of lines but in a denser form. For
1722
instance, if lines is a list of 1000 bytestrings each ending in \n,
1723
dense_lines may be a list with one line in it, containing all the
1724
1000's lines and their \n's. Using dense_lines if it is already
1725
known is a win because the string join to create bytes in this
1726
function spends less time resizing the final string.
1727
:return: (len, a StringIO instance with the raw data ready to read.)
1729
# Note: using a string copy here increases memory pressure with e.g.
1730
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1731
# when doing the initial commit of a mozilla tree. RBC 20070921
1732
bytes = ''.join(chain(
1733
["version %s %d %s\n" % (key[-1],
1736
dense_lines or lines,
1737
["end %s\n" % key[-1]]))
1738
if type(bytes) != str:
1739
raise AssertionError(
1740
'data must be plain bytes was %s' % type(bytes))
1741
if lines and lines[-1][-1] != '\n':
1742
raise ValueError('corrupt lines value %r' % lines)
1743
compressed_bytes = tuned_gzip.bytes_to_gzip(bytes)
1744
return len(compressed_bytes), compressed_bytes
1746
def _split_header(self, line):
1749
raise KnitCorrupt(self,
1750
'unexpected number of elements in record header')
1754
"""See VersionedFiles.keys."""
1755
if 'evil' in debug.debug_flags:
1756
trace.mutter_callsite(2, "keys scales with size of history")
1757
sources = [self._index] + self._fallback_vfs
1759
for source in sources:
1760
result.update(source.keys())
1764
class _KndxIndex(object):
1765
"""Manages knit index files
1767
The index is kept in memory and read on startup, to enable
1768
fast lookups of revision information. The cursor of the index
1769
file is always pointing to the end, making it easy to append
1772
_cache is a cache for fast mapping from version id to a Index
1775
_history is a cache for fast mapping from indexes to version ids.
1777
The index data format is dictionary compressed when it comes to
1778
parent references; a index entry may only have parents that with a
1779
lover index number. As a result, the index is topological sorted.
1781
Duplicate entries may be written to the index for a single version id
1782
if this is done then the latter one completely replaces the former:
1783
this allows updates to correct version and parent information.
1784
Note that the two entries may share the delta, and that successive
1785
annotations and references MUST point to the first entry.
1787
The index file on disc contains a header, followed by one line per knit
1788
record. The same revision can be present in an index file more than once.
1789
The first occurrence gets assigned a sequence number starting from 0.
1791
The format of a single line is
1792
REVISION_ID FLAGS BYTE_OFFSET LENGTH( PARENT_ID|PARENT_SEQUENCE_ID)* :\n
1793
REVISION_ID is a utf8-encoded revision id
1794
FLAGS is a comma separated list of flags about the record. Values include
1795
no-eol, line-delta, fulltext.
1796
BYTE_OFFSET is the ascii representation of the byte offset in the data file
1797
that the the compressed data starts at.
1798
LENGTH is the ascii representation of the length of the data file.
1799
PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
1801
PARENT_SEQUENCE_ID the ascii representation of the sequence number of a
1802
revision id already in the knit that is a parent of REVISION_ID.
1803
The ' :' marker is the end of record marker.
1806
when a write is interrupted to the index file, it will result in a line
1807
that does not end in ' :'. If the ' :' is not present at the end of a line,
1808
or at the end of the file, then the record that is missing it will be
1809
ignored by the parser.
1811
When writing new records to the index file, the data is preceded by '\n'
1812
to ensure that records always start on new lines even if the last write was
1813
interrupted. As a result its normal for the last line in the index to be
1814
missing a trailing newline. One can be added with no harmful effects.
1816
:ivar _kndx_cache: dict from prefix to the old state of KnitIndex objects,
1817
where prefix is e.g. the (fileid,) for .texts instances or () for
1818
constant-mapped things like .revisions, and the old state is
1819
tuple(history_vector, cache_dict). This is used to prevent having an
1820
ABI change with the C extension that reads .kndx files.
1823
HEADER = "# bzr knit index 8\n"
1825
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1826
"""Create a _KndxIndex on transport using mapper."""
1827
self._transport = transport
1828
self._mapper = mapper
1829
self._get_scope = get_scope
1830
self._allow_writes = allow_writes
1831
self._is_locked = is_locked
1833
self.has_graph = True
1835
def add_records(self, records, random_id=False):
1836
"""Add multiple records to the index.
1838
:param records: a list of tuples:
1839
(key, options, access_memo, parents).
1840
:param random_id: If True the ids being added were randomly generated
1841
and no check for existence will be performed.
1844
for record in records:
1847
path = self._mapper.map(key) + '.kndx'
1848
path_keys = paths.setdefault(path, (prefix, []))
1849
path_keys[1].append(record)
1850
for path in sorted(paths):
1851
prefix, path_keys = paths[path]
1852
self._load_prefixes([prefix])
1854
orig_history = self._kndx_cache[prefix][1][:]
1855
orig_cache = self._kndx_cache[prefix][0].copy()
1858
for key, options, (_, pos, size), parents in path_keys:
1860
# kndx indices cannot be parentless.
1862
line = "\n%s %s %s %s %s :" % (
1863
key[-1], ','.join(options), pos, size,
1864
self._dictionary_compress(parents))
1865
if type(line) != str:
1866
raise AssertionError(
1867
'data must be utf8 was %s' % type(line))
1869
self._cache_key(key, options, pos, size, parents)
1870
if len(orig_history):
1871
self._transport.append_bytes(path, ''.join(lines))
1873
self._init_index(path, lines)
1875
# If any problems happen, restore the original values and re-raise
1876
self._kndx_cache[prefix] = (orig_cache, orig_history)
1879
def _cache_key(self, key, options, pos, size, parent_keys):
1880
"""Cache a version record in the history array and index cache.
1882
This is inlined into _load_data for performance. KEEP IN SYNC.
1883
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1887
version_id = key[-1]
1888
# last-element only for compatibilty with the C load_data.
1889
parents = tuple(parent[-1] for parent in parent_keys)
1890
for parent in parent_keys:
1891
if parent[:-1] != prefix:
1892
raise ValueError("mismatched prefixes for %r, %r" % (
1894
cache, history = self._kndx_cache[prefix]
1895
# only want the _history index to reference the 1st index entry
1897
if version_id not in cache:
1898
index = len(history)
1899
history.append(version_id)
1901
index = cache[version_id][5]
1902
cache[version_id] = (version_id,
1909
def check_header(self, fp):
1910
line = fp.readline()
1912
# An empty file can actually be treated as though the file doesn't
1914
raise errors.NoSuchFile(self)
1915
if line != self.HEADER:
1916
raise KnitHeaderError(badline=line, filename=self)
1918
def _check_read(self):
1919
if not self._is_locked():
1920
raise errors.ObjectNotLocked(self)
1921
if self._get_scope() != self._scope:
1924
def _check_write_ok(self):
1925
"""Assert if not writes are permitted."""
1926
if not self._is_locked():
1927
raise errors.ObjectNotLocked(self)
1928
if self._get_scope() != self._scope:
1930
if self._mode != 'w':
1931
raise errors.ReadOnlyObjectDirtiedError(self)
1933
def get_build_details(self, keys):
1934
"""Get the method, index_memo and compression parent for keys.
1936
Ghosts are omitted from the result.
1938
:param keys: An iterable of keys.
1939
:return: A dict of key:(index_memo, compression_parent, parents,
1942
opaque structure to pass to read_records to extract the raw
1945
Content that this record is built upon, may be None
1947
Logical parents of this node
1949
extra information about the content which needs to be passed to
1950
Factory.parse_record
1952
parent_map = self.get_parent_map(keys)
1955
if key not in parent_map:
1957
method = self.get_method(key)
1958
parents = parent_map[key]
1959
if method == 'fulltext':
1960
compression_parent = None
1962
compression_parent = parents[0]
1963
noeol = 'no-eol' in self.get_options(key)
1964
index_memo = self.get_position(key)
1965
result[key] = (index_memo, compression_parent,
1966
parents, (method, noeol))
1969
def get_method(self, key):
1970
"""Return compression method of specified key."""
1971
options = self.get_options(key)
1972
if 'fulltext' in options:
1974
elif 'line-delta' in options:
1977
raise errors.KnitIndexUnknownMethod(self, options)
1979
def get_options(self, key):
1980
"""Return a list representing options.
1984
prefix, suffix = self._split_key(key)
1985
self._load_prefixes([prefix])
1987
return self._kndx_cache[prefix][0][suffix][1]
1989
raise RevisionNotPresent(key, self)
1991
def get_parent_map(self, keys):
1992
"""Get a map of the parents of keys.
1994
:param keys: The keys to look up parents for.
1995
:return: A mapping from keys to parents. Absent keys are absent from
1998
# Parse what we need to up front, this potentially trades off I/O
1999
# locality (.kndx and .knit in the same block group for the same file
2000
# id) for less checking in inner loops.
2001
prefixes = set(key[:-1] for key in keys)
2002
self._load_prefixes(prefixes)
2007
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
2011
result[key] = tuple(prefix + (suffix,) for
2012
suffix in suffix_parents)
2015
def get_position(self, key):
2016
"""Return details needed to access the version.
2018
:return: a tuple (key, data position, size) to hand to the access
2019
logic to get the record.
2021
prefix, suffix = self._split_key(key)
2022
self._load_prefixes([prefix])
2023
entry = self._kndx_cache[prefix][0][suffix]
2024
return key, entry[2], entry[3]
2026
has_key = _mod_index._has_key_from_parent_map
2028
def _init_index(self, path, extra_lines=[]):
2029
"""Initialize an index."""
2031
sio.write(self.HEADER)
2032
sio.writelines(extra_lines)
2034
self._transport.put_file_non_atomic(path, sio,
2035
create_parent_dir=True)
2036
# self._create_parent_dir)
2037
# mode=self._file_mode,
2038
# dir_mode=self._dir_mode)
2041
"""Get all the keys in the collection.
2043
The keys are not ordered.
2046
# Identify all key prefixes.
2047
# XXX: A bit hacky, needs polish.
2048
if type(self._mapper) == ConstantMapper:
2052
for quoted_relpath in self._transport.iter_files_recursive():
2053
path, ext = os.path.splitext(quoted_relpath)
2055
prefixes = [self._mapper.unmap(path) for path in relpaths]
2056
self._load_prefixes(prefixes)
2057
for prefix in prefixes:
2058
for suffix in self._kndx_cache[prefix][1]:
2059
result.add(prefix + (suffix,))
2062
def _load_prefixes(self, prefixes):
2063
"""Load the indices for prefixes."""
2065
for prefix in prefixes:
2066
if prefix not in self._kndx_cache:
2067
# the load_data interface writes to these variables.
2070
self._filename = prefix
2072
path = self._mapper.map(prefix) + '.kndx'
2073
fp = self._transport.get(path)
2075
# _load_data may raise NoSuchFile if the target knit is
2077
_load_data(self, fp)
2080
self._kndx_cache[prefix] = (self._cache, self._history)
2085
self._kndx_cache[prefix] = ({}, [])
2086
if type(self._mapper) == ConstantMapper:
2087
# preserve behaviour for revisions.kndx etc.
2088
self._init_index(path)
2093
missing_keys = _mod_index._missing_keys_from_parent_map
2095
def _partition_keys(self, keys):
2096
"""Turn keys into a dict of prefix:suffix_list."""
2099
prefix_keys = result.setdefault(key[:-1], [])
2100
prefix_keys.append(key[-1])
2103
def _dictionary_compress(self, keys):
2104
"""Dictionary compress keys.
2106
:param keys: The keys to generate references to.
2107
:return: A string representation of keys. keys which are present are
2108
dictionary compressed, and others are emitted as fulltext with a
2114
prefix = keys[0][:-1]
2115
cache = self._kndx_cache[prefix][0]
2117
if key[:-1] != prefix:
2118
# kndx indices cannot refer across partitioned storage.
2119
raise ValueError("mismatched prefixes for %r" % keys)
2120
if key[-1] in cache:
2121
# -- inlined lookup() --
2122
result_list.append(str(cache[key[-1]][5]))
2123
# -- end lookup () --
2125
result_list.append('.' + key[-1])
2126
return ' '.join(result_list)
2128
def _reset_cache(self):
2129
# Possibly this should be a LRU cache. A dictionary from key_prefix to
2130
# (cache_dict, history_vector) for parsed kndx files.
2131
self._kndx_cache = {}
2132
self._scope = self._get_scope()
2133
allow_writes = self._allow_writes()
2139
def _sort_keys_by_io(self, keys, positions):
2140
"""Figure out an optimal order to read the records for the given keys.
2142
Sort keys, grouped by index and sorted by position.
2144
:param keys: A list of keys whose records we want to read. This will be
2146
:param positions: A dict, such as the one returned by
2147
_get_components_positions()
2150
def get_sort_key(key):
2151
index_memo = positions[key][1]
2152
# Group by prefix and position. index_memo[0] is the key, so it is
2153
# (file_id, revision_id) and we don't want to sort on revision_id,
2154
# index_memo[1] is the position, and index_memo[2] is the size,
2155
# which doesn't matter for the sort
2156
return index_memo[0][:-1], index_memo[1]
2157
return keys.sort(key=get_sort_key)
2159
def _split_key(self, key):
2160
"""Split key into a prefix and suffix."""
2161
return key[:-1], key[-1]
2164
class _KnitGraphIndex(object):
2165
"""A KnitVersionedFiles index layered on GraphIndex."""
2167
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2169
"""Construct a KnitGraphIndex on a graph_index.
2171
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2172
:param is_locked: A callback to check whether the object should answer
2174
:param deltas: Allow delta-compressed records.
2175
:param parents: If True, record knits parents, if not do not record
2177
:param add_callback: If not None, allow additions to the index and call
2178
this callback with a list of added GraphIndex nodes:
2179
[(node, value, node_refs), ...]
2180
:param is_locked: A callback, returns True if the index is locked and
2183
self._add_callback = add_callback
2184
self._graph_index = graph_index
2185
self._deltas = deltas
2186
self._parents = parents
2187
if deltas and not parents:
2188
# XXX: TODO: Delta tree and parent graph should be conceptually
2190
raise KnitCorrupt(self, "Cannot do delta compression without "
2192
self.has_graph = parents
2193
self._is_locked = is_locked
2196
return "%s(%r)" % (self.__class__.__name__, self._graph_index)
2198
def add_records(self, records, random_id=False):
2199
"""Add multiple records to the index.
2201
This function does not insert data into the Immutable GraphIndex
2202
backing the KnitGraphIndex, instead it prepares data for insertion by
2203
the caller and checks that it is safe to insert then calls
2204
self._add_callback with the prepared GraphIndex nodes.
2206
:param records: a list of tuples:
2207
(key, options, access_memo, parents).
2208
:param random_id: If True the ids being added were randomly generated
2209
and no check for existence will be performed.
2211
if not self._add_callback:
2212
raise errors.ReadOnlyError(self)
2213
# we hope there are no repositories with inconsistent parentage
2217
for (key, options, access_memo, parents) in records:
2219
parents = tuple(parents)
2220
index, pos, size = access_memo
2221
if 'no-eol' in options:
2225
value += "%d %d" % (pos, size)
2226
if not self._deltas:
2227
if 'line-delta' in options:
2228
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
2231
if 'line-delta' in options:
2232
node_refs = (parents, (parents[0],))
2234
node_refs = (parents, ())
2236
node_refs = (parents, )
2239
raise KnitCorrupt(self, "attempt to add node with parents "
2240
"in parentless index.")
2242
keys[key] = (value, node_refs)
2245
present_nodes = self._get_entries(keys)
2246
for (index, key, value, node_refs) in present_nodes:
2247
if (value[0] != keys[key][0][0] or
2248
node_refs != keys[key][1]):
2249
raise KnitCorrupt(self, "inconsistent details in add_records"
2250
": %s %s" % ((value, node_refs), keys[key]))
2254
for key, (value, node_refs) in keys.iteritems():
2255
result.append((key, value, node_refs))
2257
for key, (value, node_refs) in keys.iteritems():
2258
result.append((key, value))
2259
self._add_callback(result)
2261
def _check_read(self):
2262
"""raise if reads are not permitted."""
2263
if not self._is_locked():
2264
raise errors.ObjectNotLocked(self)
2266
def _check_write_ok(self):
2267
"""Assert if writes are not permitted."""
2268
if not self._is_locked():
2269
raise errors.ObjectNotLocked(self)
2271
def _compression_parent(self, an_entry):
2272
# return the key that an_entry is compressed against, or None
2273
# Grab the second parent list (as deltas implies parents currently)
2274
compression_parents = an_entry[3][1]
2275
if not compression_parents:
2277
if len(compression_parents) != 1:
2278
raise AssertionError(
2279
"Too many compression parents: %r" % compression_parents)
2280
return compression_parents[0]
2282
def get_build_details(self, keys):
2283
"""Get the method, index_memo and compression parent for version_ids.
2285
Ghosts are omitted from the result.
2287
:param keys: An iterable of keys.
2288
:return: A dict of key:
2289
(index_memo, compression_parent, parents, record_details).
2291
opaque structure to pass to read_records to extract the raw
2294
Content that this record is built upon, may be None
2296
Logical parents of this node
2298
extra information about the content which needs to be passed to
2299
Factory.parse_record
2303
entries = self._get_entries(keys, False)
2304
for entry in entries:
2306
if not self._parents:
2309
parents = entry[3][0]
2310
if not self._deltas:
2311
compression_parent_key = None
2313
compression_parent_key = self._compression_parent(entry)
2314
noeol = (entry[2][0] == 'N')
2315
if compression_parent_key:
2316
method = 'line-delta'
2319
result[key] = (self._node_to_position(entry),
2320
compression_parent_key, parents,
2324
def _get_entries(self, keys, check_present=False):
2325
"""Get the entries for keys.
2327
:param keys: An iterable of index key tuples.
2332
for node in self._graph_index.iter_entries(keys):
2334
found_keys.add(node[1])
2336
# adapt parentless index to the rest of the code.
2337
for node in self._graph_index.iter_entries(keys):
2338
yield node[0], node[1], node[2], ()
2339
found_keys.add(node[1])
2341
missing_keys = keys.difference(found_keys)
2343
raise RevisionNotPresent(missing_keys.pop(), self)
2345
def get_method(self, key):
2346
"""Return compression method of specified key."""
2347
return self._get_method(self._get_node(key))
2349
def _get_method(self, node):
2350
if not self._deltas:
2352
if self._compression_parent(node):
2357
def _get_node(self, key):
2359
return list(self._get_entries([key]))[0]
2361
raise RevisionNotPresent(key, self)
2363
def get_options(self, key):
2364
"""Return a list representing options.
2368
node = self._get_node(key)
2369
options = [self._get_method(node)]
2370
if node[2][0] == 'N':
2371
options.append('no-eol')
2374
def get_parent_map(self, keys):
2375
"""Get a map of the parents of keys.
2377
:param keys: The keys to look up parents for.
2378
:return: A mapping from keys to parents. Absent keys are absent from
2382
nodes = self._get_entries(keys)
2386
result[node[1]] = node[3][0]
2389
result[node[1]] = None
2392
def get_position(self, key):
2393
"""Return details needed to access the version.
2395
:return: a tuple (index, data position, size) to hand to the access
2396
logic to get the record.
2398
node = self._get_node(key)
2399
return self._node_to_position(node)
2401
has_key = _mod_index._has_key_from_parent_map
2404
"""Get all the keys in the collection.
2406
The keys are not ordered.
2409
return [node[1] for node in self._graph_index.iter_all_entries()]
2411
missing_keys = _mod_index._missing_keys_from_parent_map
2413
def _node_to_position(self, node):
2414
"""Convert an index value to position details."""
2415
bits = node[2][1:].split(' ')
2416
return node[0], int(bits[0]), int(bits[1])
2418
def _sort_keys_by_io(self, keys, positions):
2419
"""Figure out an optimal order to read the records for the given keys.
2421
Sort keys, grouped by index and sorted by position.
2423
:param keys: A list of keys whose records we want to read. This will be
2425
:param positions: A dict, such as the one returned by
2426
_get_components_positions()
2429
def get_index_memo(key):
2430
# index_memo is at offset [1]. It is made up of (GraphIndex,
2431
# position, size). GI is an object, which will be unique for each
2432
# pack file. This causes us to group by pack file, then sort by
2433
# position. Size doesn't matter, but it isn't worth breaking up the
2435
return positions[key][1]
2436
return keys.sort(key=get_index_memo)
2439
class _KnitKeyAccess(object):
2440
"""Access to records in .knit files."""
2442
def __init__(self, transport, mapper):
2443
"""Create a _KnitKeyAccess with transport and mapper.
2445
:param transport: The transport the access object is rooted at.
2446
:param mapper: The mapper used to map keys to .knit files.
2448
self._transport = transport
2449
self._mapper = mapper
2451
def add_raw_records(self, key_sizes, raw_data):
2452
"""Add raw knit bytes to a storage area.
2454
The data is spooled to the container writer in one bytes-record per
2457
:param sizes: An iterable of tuples containing the key and size of each
2459
:param raw_data: A bytestring containing the data.
2460
:return: A list of memos to retrieve the record later. Each memo is an
2461
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2462
length), where the key is the record key.
2464
if type(raw_data) != str:
2465
raise AssertionError(
2466
'data must be plain bytes was %s' % type(raw_data))
2469
# TODO: This can be tuned for writing to sftp and other servers where
2470
# append() is relatively expensive by grouping the writes to each key
2472
for key, size in key_sizes:
2473
path = self._mapper.map(key)
2475
base = self._transport.append_bytes(path + '.knit',
2476
raw_data[offset:offset+size])
2477
except errors.NoSuchFile:
2478
self._transport.mkdir(osutils.dirname(path))
2479
base = self._transport.append_bytes(path + '.knit',
2480
raw_data[offset:offset+size])
2484
result.append((key, base, size))
2487
def get_raw_records(self, memos_for_retrieval):
2488
"""Get the raw bytes for a records.
2490
:param memos_for_retrieval: An iterable containing the access memo for
2491
retrieving the bytes.
2492
:return: An iterator over the bytes of the records.
2494
# first pass, group into same-index request to minimise readv's issued.
2496
current_prefix = None
2497
for (key, offset, length) in memos_for_retrieval:
2498
if current_prefix == key[:-1]:
2499
current_list.append((offset, length))
2501
if current_prefix is not None:
2502
request_lists.append((current_prefix, current_list))
2503
current_prefix = key[:-1]
2504
current_list = [(offset, length)]
2505
# handle the last entry
2506
if current_prefix is not None:
2507
request_lists.append((current_prefix, current_list))
2508
for prefix, read_vector in request_lists:
2509
path = self._mapper.map(prefix) + '.knit'
2510
for pos, data in self._transport.readv(path, read_vector):
2514
class _DirectPackAccess(object):
2515
"""Access to data in one or more packs with less translation."""
2517
def __init__(self, index_to_packs, reload_func=None):
2518
"""Create a _DirectPackAccess object.
2520
:param index_to_packs: A dict mapping index objects to the transport
2521
and file names for obtaining data.
2522
:param reload_func: A function to call if we determine that the pack
2523
files have moved and we need to reload our caches. See
2524
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
2526
self._container_writer = None
2527
self._write_index = None
2528
self._indices = index_to_packs
2529
self._reload_func = reload_func
2531
def add_raw_records(self, key_sizes, raw_data):
2532
"""Add raw knit bytes to a storage area.
2534
The data is spooled to the container writer in one bytes-record per
2537
:param sizes: An iterable of tuples containing the key and size of each
2539
:param raw_data: A bytestring containing the data.
2540
:return: A list of memos to retrieve the record later. Each memo is an
2541
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2542
length), where the index field is the write_index object supplied
2543
to the PackAccess object.
2545
if type(raw_data) != str:
2546
raise AssertionError(
2547
'data must be plain bytes was %s' % type(raw_data))
2550
for key, size in key_sizes:
2551
p_offset, p_length = self._container_writer.add_bytes_record(
2552
raw_data[offset:offset+size], [])
2554
result.append((self._write_index, p_offset, p_length))
2557
def get_raw_records(self, memos_for_retrieval):
2558
"""Get the raw bytes for a records.
2560
:param memos_for_retrieval: An iterable containing the (index, pos,
2561
length) memo for retrieving the bytes. The Pack access method
2562
looks up the pack to use for a given record in its index_to_pack
2564
:return: An iterator over the bytes of the records.
2566
# first pass, group into same-index requests
2568
current_index = None
2569
for (index, offset, length) in memos_for_retrieval:
2570
if current_index == index:
2571
current_list.append((offset, length))
2573
if current_index is not None:
2574
request_lists.append((current_index, current_list))
2575
current_index = index
2576
current_list = [(offset, length)]
2577
# handle the last entry
2578
if current_index is not None:
2579
request_lists.append((current_index, current_list))
2580
for index, offsets in request_lists:
2582
transport, path = self._indices[index]
2584
# A KeyError here indicates that someone has triggered an index
2585
# reload, and this index has gone missing, we need to start
2587
if self._reload_func is None:
2588
# If we don't have a _reload_func there is nothing that can
2591
raise errors.RetryWithNewPacks(reload_occurred=True,
2592
exc_info=sys.exc_info())
2594
reader = pack.make_readv_reader(transport, path, offsets)
2595
for names, read_func in reader.iter_records():
2596
yield read_func(None)
2597
except errors.NoSuchFile:
2598
# A NoSuchFile error indicates that a pack file has gone
2599
# missing on disk, we need to trigger a reload, and start over.
2600
if self._reload_func is None:
2602
raise errors.RetryWithNewPacks(reload_occurred=False,
2603
exc_info=sys.exc_info())
2605
def set_writer(self, writer, index, transport_packname):
2606
"""Set a writer to use for adding data."""
2607
if index is not None:
2608
self._indices[index] = transport_packname
2609
self._container_writer = writer
2610
self._write_index = index
2612
def reload_or_raise(self, retry_exc):
2613
"""Try calling the reload function, or re-raise the original exception.
2615
This should be called after _DirectPackAccess raises a
2616
RetryWithNewPacks exception. This function will handle the common logic
2617
of determining when the error is fatal versus being temporary.
2618
It will also make sure that the original exception is raised, rather
2619
than the RetryWithNewPacks exception.
2621
If this function returns, then the calling function should retry
2622
whatever operation was being performed. Otherwise an exception will
2625
:param retry_exc: A RetryWithNewPacks exception.
2628
if self._reload_func is None:
2630
elif not self._reload_func():
2631
# The reload claimed that nothing changed
2632
if not retry_exc.reload_occurred:
2633
# If there wasn't an earlier reload, then we really were
2634
# expecting to find changes. We didn't find them, so this is a
2638
exc_class, exc_value, exc_traceback = retry_exc.exc_info
2639
raise exc_class, exc_value, exc_traceback
2642
# Deprecated, use PatienceSequenceMatcher instead
2643
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2646
def annotate_knit(knit, revision_id):
2647
"""Annotate a knit with no cached annotations.
2649
This implementation is for knits with no cached annotations.
2650
It will work for knits with cached annotations, but this is not
2653
annotator = _KnitAnnotator(knit)
2654
return iter(annotator.annotate(revision_id))
2657
class _KnitAnnotator(object):
2658
"""Build up the annotations for a text."""
2660
def __init__(self, knit):
2663
# Content objects, differs from fulltexts because of how final newlines
2664
# are treated by knits. the content objects here will always have a
2666
self._fulltext_contents = {}
2668
# Annotated lines of specific revisions
2669
self._annotated_lines = {}
2671
# Track the raw data for nodes that we could not process yet.
2672
# This maps the revision_id of the base to a list of children that will
2673
# annotated from it.
2674
self._pending_children = {}
2676
# Nodes which cannot be extracted
2677
self._ghosts = set()
2679
# Track how many children this node has, so we know if we need to keep
2681
self._annotate_children = {}
2682
self._compression_children = {}
2684
self._all_build_details = {}
2685
# The children => parent revision_id graph
2686
self._revision_id_graph = {}
2688
self._heads_provider = None
2690
self._nodes_to_keep_annotations = set()
2691
self._generations_until_keep = 100
2693
def set_generations_until_keep(self, value):
2694
"""Set the number of generations before caching a node.
2696
Setting this to -1 will cache every merge node, setting this higher
2697
will cache fewer nodes.
2699
self._generations_until_keep = value
2701
def _add_fulltext_content(self, revision_id, content_obj):
2702
self._fulltext_contents[revision_id] = content_obj
2703
# TODO: jam 20080305 It might be good to check the sha1digest here
2704
return content_obj.text()
2706
def _check_parents(self, child, nodes_to_annotate):
2707
"""Check if all parents have been processed.
2709
:param child: A tuple of (rev_id, parents, raw_content)
2710
:param nodes_to_annotate: If child is ready, add it to
2711
nodes_to_annotate, otherwise put it back in self._pending_children
2713
for parent_id in child[1]:
2714
if (parent_id not in self._annotated_lines):
2715
# This parent is present, but another parent is missing
2716
self._pending_children.setdefault(parent_id,
2720
# This one is ready to be processed
2721
nodes_to_annotate.append(child)
2723
def _add_annotation(self, revision_id, fulltext, parent_ids,
2724
left_matching_blocks=None):
2725
"""Add an annotation entry.
2727
All parents should already have been annotated.
2728
:return: A list of children that now have their parents satisfied.
2730
a = self._annotated_lines
2731
annotated_parent_lines = [a[p] for p in parent_ids]
2732
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2733
fulltext, revision_id, left_matching_blocks,
2734
heads_provider=self._get_heads_provider()))
2735
self._annotated_lines[revision_id] = annotated_lines
2736
for p in parent_ids:
2737
ann_children = self._annotate_children[p]
2738
ann_children.remove(revision_id)
2739
if (not ann_children
2740
and p not in self._nodes_to_keep_annotations):
2741
del self._annotated_lines[p]
2742
del self._all_build_details[p]
2743
if p in self._fulltext_contents:
2744
del self._fulltext_contents[p]
2745
# Now that we've added this one, see if there are any pending
2746
# deltas to be done, certainly this parent is finished
2747
nodes_to_annotate = []
2748
for child in self._pending_children.pop(revision_id, []):
2749
self._check_parents(child, nodes_to_annotate)
2750
return nodes_to_annotate
2752
def _get_build_graph(self, key):
2753
"""Get the graphs for building texts and annotations.
2755
The data you need for creating a full text may be different than the
2756
data you need to annotate that text. (At a minimum, you need both
2757
parents to create an annotation, but only need 1 parent to generate the
2760
:return: A list of (key, index_memo) records, suitable for
2761
passing to read_records_iter to start reading in the raw data fro/
2764
if key in self._annotated_lines:
2767
pending = set([key])
2772
# get all pending nodes
2774
this_iteration = pending
2775
build_details = self._knit._index.get_build_details(this_iteration)
2776
self._all_build_details.update(build_details)
2777
# new_nodes = self._knit._index._get_entries(this_iteration)
2779
for key, details in build_details.iteritems():
2780
(index_memo, compression_parent, parents,
2781
record_details) = details
2782
self._revision_id_graph[key] = parents
2783
records.append((key, index_memo))
2784
# Do we actually need to check _annotated_lines?
2785
pending.update(p for p in parents
2786
if p not in self._all_build_details)
2787
if compression_parent:
2788
self._compression_children.setdefault(compression_parent,
2791
for parent in parents:
2792
self._annotate_children.setdefault(parent,
2794
num_gens = generation - kept_generation
2795
if ((num_gens >= self._generations_until_keep)
2796
and len(parents) > 1):
2797
kept_generation = generation
2798
self._nodes_to_keep_annotations.add(key)
2800
missing_versions = this_iteration.difference(build_details.keys())
2801
self._ghosts.update(missing_versions)
2802
for missing_version in missing_versions:
2803
# add a key, no parents
2804
self._revision_id_graph[missing_version] = ()
2805
pending.discard(missing_version) # don't look for it
2806
if self._ghosts.intersection(self._compression_children):
2808
"We cannot have nodes which have a ghost compression parent:\n"
2810
"compression children: %r"
2811
% (self._ghosts, self._compression_children))
2812
# Cleanout anything that depends on a ghost so that we don't wait for
2813
# the ghost to show up
2814
for node in self._ghosts:
2815
if node in self._annotate_children:
2816
# We won't be building this node
2817
del self._annotate_children[node]
2818
# Generally we will want to read the records in reverse order, because
2819
# we find the parent nodes after the children
2823
def _annotate_records(self, records):
2824
"""Build the annotations for the listed records."""
2825
# We iterate in the order read, rather than a strict order requested
2826
# However, process what we can, and put off to the side things that
2827
# still need parents, cleaning them up when those parents are
2829
for (rev_id, record,
2830
digest) in self._knit._read_records_iter(records):
2831
if rev_id in self._annotated_lines:
2833
parent_ids = self._revision_id_graph[rev_id]
2834
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2835
details = self._all_build_details[rev_id]
2836
(index_memo, compression_parent, parents,
2837
record_details) = details
2838
nodes_to_annotate = []
2839
# TODO: Remove the punning between compression parents, and
2840
# parent_ids, we should be able to do this without assuming
2842
if len(parent_ids) == 0:
2843
# There are no parents for this node, so just add it
2844
# TODO: This probably needs to be decoupled
2845
fulltext_content, delta = self._knit._factory.parse_record(
2846
rev_id, record, record_details, None)
2847
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2848
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2849
parent_ids, left_matching_blocks=None))
2851
child = (rev_id, parent_ids, record)
2852
# Check if all the parents are present
2853
self._check_parents(child, nodes_to_annotate)
2854
while nodes_to_annotate:
2855
# Should we use a queue here instead of a stack?
2856
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2857
(index_memo, compression_parent, parents,
2858
record_details) = self._all_build_details[rev_id]
2860
if compression_parent is not None:
2861
comp_children = self._compression_children[compression_parent]
2862
if rev_id not in comp_children:
2863
raise AssertionError("%r not in compression children %r"
2864
% (rev_id, comp_children))
2865
# If there is only 1 child, it is safe to reuse this
2867
reuse_content = (len(comp_children) == 1
2868
and compression_parent not in
2869
self._nodes_to_keep_annotations)
2871
# Remove it from the cache since it will be changing
2872
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2873
# Make sure to copy the fulltext since it might be
2875
parent_fulltext = list(parent_fulltext_content.text())
2877
parent_fulltext_content = self._fulltext_contents[compression_parent]
2878
parent_fulltext = parent_fulltext_content.text()
2879
comp_children.remove(rev_id)
2880
fulltext_content, delta = self._knit._factory.parse_record(
2881
rev_id, record, record_details,
2882
parent_fulltext_content,
2883
copy_base_content=(not reuse_content))
2884
fulltext = self._add_fulltext_content(rev_id,
2886
if compression_parent == parent_ids[0]:
2887
# the compression_parent is the left parent, so we can
2889
blocks = KnitContent.get_line_delta_blocks(delta,
2890
parent_fulltext, fulltext)
2892
fulltext_content = self._knit._factory.parse_fulltext(
2894
fulltext = self._add_fulltext_content(rev_id,
2896
nodes_to_annotate.extend(
2897
self._add_annotation(rev_id, fulltext, parent_ids,
2898
left_matching_blocks=blocks))
2900
def _get_heads_provider(self):
2901
"""Create a heads provider for resolving ancestry issues."""
2902
if self._heads_provider is not None:
2903
return self._heads_provider
2904
parent_provider = _mod_graph.DictParentsProvider(
2905
self._revision_id_graph)
2906
graph_obj = _mod_graph.Graph(parent_provider)
2907
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2908
self._heads_provider = head_cache
2911
def annotate(self, key):
2912
"""Return the annotated fulltext at the given key.
2914
:param key: The key to annotate.
2916
if len(self._knit._fallback_vfs) > 0:
2917
# stacked knits can't use the fast path at present.
2918
return self._simple_annotate(key)
2921
records = self._get_build_graph(key)
2922
if key in self._ghosts:
2923
raise errors.RevisionNotPresent(key, self._knit)
2924
self._annotate_records(records)
2925
return self._annotated_lines[key]
2926
except errors.RetryWithNewPacks, e:
2927
self._knit._access.reload_or_raise(e)
2928
# The cached build_details are no longer valid
2929
self._all_build_details.clear()
2931
def _simple_annotate(self, key):
2932
"""Return annotated fulltext, rediffing from the full texts.
2934
This is slow but makes no assumptions about the repository
2935
being able to produce line deltas.
2937
# TODO: this code generates a parent maps of present ancestors; it
2938
# could be split out into a separate method, and probably should use
2939
# iter_ancestry instead. -- mbp and robertc 20080704
2940
graph = _mod_graph.Graph(self._knit)
2941
head_cache = _mod_graph.FrozenHeadsCache(graph)
2942
search = graph._make_breadth_first_searcher([key])
2946
present, ghosts = search.next_with_ghosts()
2947
except StopIteration:
2949
keys.update(present)
2950
parent_map = self._knit.get_parent_map(keys)
2952
reannotate = annotate.reannotate
2953
for record in self._knit.get_record_stream(keys, 'topological', True):
2955
fulltext = split_lines(record.get_bytes_as('fulltext'))
2956
parents = parent_map[key]
2957
if parents is not None:
2958
parent_lines = [parent_cache[parent] for parent in parent_map[key]]
2961
parent_cache[key] = list(
2962
reannotate(parent_lines, fulltext, key, None, head_cache))
2964
return parent_cache[key]
2966
raise errors.RevisionNotPresent(key, self._knit)
2970
from bzrlib._knit_load_data_c import _load_data_c as _load_data
2972
from bzrlib._knit_load_data_py import _load_data_py as _load_data
added
removed
bzrlib/knit.py
