1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Knit versionedfile implementation.
19
A knit is a versioned file implementation that supports efficient append only
23
lifeless: the data file is made up of "delta records". each delta record has a delta header
24
that contains; (1) a version id, (2) the size of the delta (in lines), and (3) the digest of
25
the -expanded data- (ie, the delta applied to the parent). the delta also ends with a
26
end-marker; simply "end VERSION"
28
delta can be line or full contents.a
29
... the 8's there are the index number of the annotation.
30
version robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad 7 c7d23b2a5bd6ca00e8e266cec0ec228158ee9f9e
34
8 e.set('executable', 'yes')
36
8 if elt.get('executable') == 'yes':
37
8 ie.executable = True
38
end robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad
42
09:33 < jrydberg> lifeless: each index is made up of a tuple of; version id, options, position, size, parents
43
09:33 < jrydberg> lifeless: the parents are currently dictionary compressed
44
09:33 < jrydberg> lifeless: (meaning it currently does not support ghosts)
45
09:33 < lifeless> right
46
09:33 < jrydberg> lifeless: the position and size is the range in the data file
49
so the index sequence is the dictionary compressed sequence number used
50
in the deltas to provide line annotation
55
# 10:16 < lifeless> make partial index writes safe
56
# 10:16 < lifeless> implement 'knit.check()' like weave.check()
57
# 10:17 < lifeless> record known ghosts so we can detect when they are filled in rather than the current 'reweave
59
# move sha1 out of the content so that join is faster at verifying parents
60
# record content length ?
63
from cStringIO import StringIO
64
from itertools import izip, chain
69
from bzrlib.lazy_import import lazy_import
70
lazy_import(globals(), """
90
from bzrlib.errors import (
98
RevisionAlreadyPresent,
101
from bzrlib.osutils import (
108
from bzrlib.versionedfile import (
109
AbsentContentFactory,
113
ChunkedContentFactory,
119
# TODO: Split out code specific to this format into an associated object.
121
# TODO: Can we put in some kind of value to check that the index and data
122
# files belong together?
124
# TODO: accommodate binaries, perhaps by storing a byte count
126
# TODO: function to check whole file
128
# TODO: atomically append data, then measure backwards from the cursor
129
# position after writing to work out where it was located. we may need to
130
# bypass python file buffering.
132
DATA_SUFFIX = '.knit'
133
INDEX_SUFFIX = '.kndx'
136
class KnitAdapter(object):
137
"""Base class for knit record adaption."""
139
def __init__(self, basis_vf):
140
"""Create an adapter which accesses full texts from basis_vf.
142
:param basis_vf: A versioned file to access basis texts of deltas from.
143
May be None for adapters that do not need to access basis texts.
145
self._data = KnitVersionedFiles(None, None)
146
self._annotate_factory = KnitAnnotateFactory()
147
self._plain_factory = KnitPlainFactory()
148
self._basis_vf = basis_vf
151
class FTAnnotatedToUnannotated(KnitAdapter):
152
"""An adapter from FT annotated knits to unannotated ones."""
154
def get_bytes(self, factory, annotated_compressed_bytes):
156
self._data._parse_record_unchecked(annotated_compressed_bytes)
157
content = self._annotate_factory.parse_fulltext(contents, rec[1])
158
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
162
class DeltaAnnotatedToUnannotated(KnitAdapter):
163
"""An adapter for deltas from annotated to unannotated."""
165
def get_bytes(self, factory, annotated_compressed_bytes):
167
self._data._parse_record_unchecked(annotated_compressed_bytes)
168
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
170
contents = self._plain_factory.lower_line_delta(delta)
171
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
175
class FTAnnotatedToFullText(KnitAdapter):
176
"""An adapter from FT annotated knits to unannotated ones."""
178
def get_bytes(self, factory, annotated_compressed_bytes):
180
self._data._parse_record_unchecked(annotated_compressed_bytes)
181
content, delta = self._annotate_factory.parse_record(factory.key[-1],
182
contents, factory._build_details, None)
183
return ''.join(content.text())
186
class DeltaAnnotatedToFullText(KnitAdapter):
187
"""An adapter for deltas from annotated to unannotated."""
189
def get_bytes(self, factory, annotated_compressed_bytes):
191
self._data._parse_record_unchecked(annotated_compressed_bytes)
192
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
194
compression_parent = factory.parents[0]
195
basis_entry = self._basis_vf.get_record_stream(
196
[compression_parent], 'unordered', True).next()
197
if basis_entry.storage_kind == 'absent':
198
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
199
basis_chunks = basis_entry.get_bytes_as('chunked')
200
basis_lines = osutils.chunks_to_lines(basis_chunks)
201
# Manually apply the delta because we have one annotated content and
203
basis_content = PlainKnitContent(basis_lines, compression_parent)
204
basis_content.apply_delta(delta, rec[1])
205
basis_content._should_strip_eol = factory._build_details[1]
206
return ''.join(basis_content.text())
209
class FTPlainToFullText(KnitAdapter):
210
"""An adapter from FT plain knits to unannotated ones."""
212
def get_bytes(self, factory, compressed_bytes):
214
self._data._parse_record_unchecked(compressed_bytes)
215
content, delta = self._plain_factory.parse_record(factory.key[-1],
216
contents, factory._build_details, None)
217
return ''.join(content.text())
220
class DeltaPlainToFullText(KnitAdapter):
221
"""An adapter for deltas from annotated to unannotated."""
223
def get_bytes(self, factory, compressed_bytes):
225
self._data._parse_record_unchecked(compressed_bytes)
226
delta = self._plain_factory.parse_line_delta(contents, rec[1])
227
compression_parent = factory.parents[0]
228
# XXX: string splitting overhead.
229
basis_entry = self._basis_vf.get_record_stream(
230
[compression_parent], 'unordered', True).next()
231
if basis_entry.storage_kind == 'absent':
232
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
233
basis_chunks = basis_entry.get_bytes_as('chunked')
234
basis_lines = osutils.chunks_to_lines(basis_chunks)
235
basis_content = PlainKnitContent(basis_lines, compression_parent)
236
# Manually apply the delta because we have one annotated content and
238
content, _ = self._plain_factory.parse_record(rec[1], contents,
239
factory._build_details, basis_content)
240
return ''.join(content.text())
243
class KnitContentFactory(ContentFactory):
244
"""Content factory for streaming from knits.
246
:seealso ContentFactory:
249
def __init__(self, key, parents, build_details, sha1, raw_record,
250
annotated, knit=None):
251
"""Create a KnitContentFactory for key.
254
:param parents: The parents.
255
:param build_details: The build details as returned from
257
:param sha1: The sha1 expected from the full text of this object.
258
:param raw_record: The bytes of the knit data from disk.
259
:param annotated: True if the raw data is annotated.
261
ContentFactory.__init__(self)
264
self.parents = parents
265
if build_details[0] == 'line-delta':
270
annotated_kind = 'annotated-'
273
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
274
self._raw_record = raw_record
275
self._build_details = build_details
278
def get_bytes_as(self, storage_kind):
279
if storage_kind == self.storage_kind:
280
return self._raw_record
281
if self._knit is not None:
282
if storage_kind == 'chunked':
283
return self._knit.get_lines(self.key[0])
284
elif storage_kind == 'fulltext':
285
return self._knit.get_text(self.key[0])
286
raise errors.UnavailableRepresentation(self.key, storage_kind,
290
class KnitContent(object):
291
"""Content of a knit version to which deltas can be applied.
293
This is always stored in memory as a list of lines with \n at the end,
294
plus a flag saying if the final ending is really there or not, because that
295
corresponds to the on-disk knit representation.
299
self._should_strip_eol = False
301
def apply_delta(self, delta, new_version_id):
302
"""Apply delta to this object to become new_version_id."""
303
raise NotImplementedError(self.apply_delta)
305
def line_delta_iter(self, new_lines):
306
"""Generate line-based delta from this content to new_lines."""
307
new_texts = new_lines.text()
308
old_texts = self.text()
309
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
310
for tag, i1, i2, j1, j2 in s.get_opcodes():
313
# ofrom, oto, length, data
314
yield i1, i2, j2 - j1, new_lines._lines[j1:j2]
316
def line_delta(self, new_lines):
317
return list(self.line_delta_iter(new_lines))
320
def get_line_delta_blocks(knit_delta, source, target):
321
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
322
target_len = len(target)
325
for s_begin, s_end, t_len, new_text in knit_delta:
326
true_n = s_begin - s_pos
329
# knit deltas do not provide reliable info about whether the
330
# last line of a file matches, due to eol handling.
331
if source[s_pos + n -1] != target[t_pos + n -1]:
334
yield s_pos, t_pos, n
335
t_pos += t_len + true_n
337
n = target_len - t_pos
339
if source[s_pos + n -1] != target[t_pos + n -1]:
342
yield s_pos, t_pos, n
343
yield s_pos + (target_len - t_pos), target_len, 0
346
class AnnotatedKnitContent(KnitContent):
347
"""Annotated content."""
349
def __init__(self, lines):
350
KnitContent.__init__(self)
354
"""Return a list of (origin, text) for each content line."""
355
lines = self._lines[:]
356
if self._should_strip_eol:
357
origin, last_line = lines[-1]
358
lines[-1] = (origin, last_line.rstrip('\n'))
361
def apply_delta(self, delta, new_version_id):
362
"""Apply delta to this object to become new_version_id."""
365
for start, end, count, delta_lines in delta:
366
lines[offset+start:offset+end] = delta_lines
367
offset = offset + (start - end) + count
371
lines = [text for origin, text in self._lines]
372
except ValueError, e:
373
# most commonly (only?) caused by the internal form of the knit
374
# missing annotation information because of a bug - see thread
376
raise KnitCorrupt(self,
377
"line in annotated knit missing annotation information: %s"
379
if self._should_strip_eol:
380
lines[-1] = lines[-1].rstrip('\n')
384
return AnnotatedKnitContent(self._lines[:])
387
class PlainKnitContent(KnitContent):
388
"""Unannotated content.
390
When annotate[_iter] is called on this content, the same version is reported
391
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
395
def __init__(self, lines, version_id):
396
KnitContent.__init__(self)
398
self._version_id = version_id
401
"""Return a list of (origin, text) for each content line."""
402
return [(self._version_id, line) for line in self._lines]
404
def apply_delta(self, delta, new_version_id):
405
"""Apply delta to this object to become new_version_id."""
408
for start, end, count, delta_lines in delta:
409
lines[offset+start:offset+end] = delta_lines
410
offset = offset + (start - end) + count
411
self._version_id = new_version_id
414
return PlainKnitContent(self._lines[:], self._version_id)
418
if self._should_strip_eol:
420
lines[-1] = lines[-1].rstrip('\n')
424
class _KnitFactory(object):
425
"""Base class for common Factory functions."""
427
def parse_record(self, version_id, record, record_details,
428
base_content, copy_base_content=True):
429
"""Parse a record into a full content object.
431
:param version_id: The official version id for this content
432
:param record: The data returned by read_records_iter()
433
:param record_details: Details about the record returned by
435
:param base_content: If get_build_details returns a compression_parent,
436
you must return a base_content here, else use None
437
:param copy_base_content: When building from the base_content, decide
438
you can either copy it and return a new object, or modify it in
440
:return: (content, delta) A Content object and possibly a line-delta,
443
method, noeol = record_details
444
if method == 'line-delta':
445
if copy_base_content:
446
content = base_content.copy()
448
content = base_content
449
delta = self.parse_line_delta(record, version_id)
450
content.apply_delta(delta, version_id)
452
content = self.parse_fulltext(record, version_id)
454
content._should_strip_eol = noeol
455
return (content, delta)
458
class KnitAnnotateFactory(_KnitFactory):
459
"""Factory for creating annotated Content objects."""
463
def make(self, lines, version_id):
464
num_lines = len(lines)
465
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
467
def parse_fulltext(self, content, version_id):
468
"""Convert fulltext to internal representation
470
fulltext content is of the format
471
revid(utf8) plaintext\n
472
internal representation is of the format:
475
# TODO: jam 20070209 The tests expect this to be returned as tuples,
476
# but the code itself doesn't really depend on that.
477
# Figure out a way to not require the overhead of turning the
478
# list back into tuples.
479
lines = [tuple(line.split(' ', 1)) for line in content]
480
return AnnotatedKnitContent(lines)
482
def parse_line_delta_iter(self, lines):
483
return iter(self.parse_line_delta(lines))
485
def parse_line_delta(self, lines, version_id, plain=False):
486
"""Convert a line based delta into internal representation.
488
line delta is in the form of:
489
intstart intend intcount
491
revid(utf8) newline\n
492
internal representation is
493
(start, end, count, [1..count tuples (revid, newline)])
495
:param plain: If True, the lines are returned as a plain
496
list without annotations, not as a list of (origin, content) tuples, i.e.
497
(start, end, count, [1..count newline])
504
def cache_and_return(line):
505
origin, text = line.split(' ', 1)
506
return cache.setdefault(origin, origin), text
508
# walk through the lines parsing.
509
# Note that the plain test is explicitly pulled out of the
510
# loop to minimise any performance impact
513
start, end, count = [int(n) for n in header.split(',')]
514
contents = [next().split(' ', 1)[1] for i in xrange(count)]
515
result.append((start, end, count, contents))
518
start, end, count = [int(n) for n in header.split(',')]
519
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
520
result.append((start, end, count, contents))
523
def get_fulltext_content(self, lines):
524
"""Extract just the content lines from a fulltext."""
525
return (line.split(' ', 1)[1] for line in lines)
527
def get_linedelta_content(self, lines):
528
"""Extract just the content from a line delta.
530
This doesn't return all of the extra information stored in a delta.
531
Only the actual content lines.
536
header = header.split(',')
537
count = int(header[2])
538
for i in xrange(count):
539
origin, text = next().split(' ', 1)
542
def lower_fulltext(self, content):
543
"""convert a fulltext content record into a serializable form.
545
see parse_fulltext which this inverts.
547
# TODO: jam 20070209 We only do the caching thing to make sure that
548
# the origin is a valid utf-8 line, eventually we could remove it
549
return ['%s %s' % (o, t) for o, t in content._lines]
551
def lower_line_delta(self, delta):
552
"""convert a delta into a serializable form.
554
See parse_line_delta which this inverts.
556
# TODO: jam 20070209 We only do the caching thing to make sure that
557
# the origin is a valid utf-8 line, eventually we could remove it
559
for start, end, c, lines in delta:
560
out.append('%d,%d,%d\n' % (start, end, c))
561
out.extend(origin + ' ' + text
562
for origin, text in lines)
565
def annotate(self, knit, key):
566
content = knit._get_content(key)
567
# adjust for the fact that serialised annotations are only key suffixes
569
if type(key) == tuple:
571
origins = content.annotate()
573
for origin, line in origins:
574
result.append((prefix + (origin,), line))
577
# XXX: This smells a bit. Why would key ever be a non-tuple here?
578
# Aren't keys defined to be tuples? -- spiv 20080618
579
return content.annotate()
582
class KnitPlainFactory(_KnitFactory):
583
"""Factory for creating plain Content objects."""
587
def make(self, lines, version_id):
588
return PlainKnitContent(lines, version_id)
590
def parse_fulltext(self, content, version_id):
591
"""This parses an unannotated fulltext.
593
Note that this is not a noop - the internal representation
594
has (versionid, line) - its just a constant versionid.
596
return self.make(content, version_id)
598
def parse_line_delta_iter(self, lines, version_id):
600
num_lines = len(lines)
601
while cur < num_lines:
604
start, end, c = [int(n) for n in header.split(',')]
605
yield start, end, c, lines[cur:cur+c]
608
def parse_line_delta(self, lines, version_id):
609
return list(self.parse_line_delta_iter(lines, version_id))
611
def get_fulltext_content(self, lines):
612
"""Extract just the content lines from a fulltext."""
615
def get_linedelta_content(self, lines):
616
"""Extract just the content from a line delta.
618
This doesn't return all of the extra information stored in a delta.
619
Only the actual content lines.
624
header = header.split(',')
625
count = int(header[2])
626
for i in xrange(count):
629
def lower_fulltext(self, content):
630
return content.text()
632
def lower_line_delta(self, delta):
634
for start, end, c, lines in delta:
635
out.append('%d,%d,%d\n' % (start, end, c))
639
def annotate(self, knit, key):
640
annotator = _KnitAnnotator(knit)
641
return annotator.annotate(key)
645
def make_file_factory(annotated, mapper):
646
"""Create a factory for creating a file based KnitVersionedFiles.
648
This is only functional enough to run interface tests, it doesn't try to
649
provide a full pack environment.
651
:param annotated: knit annotations are wanted.
652
:param mapper: The mapper from keys to paths.
654
def factory(transport):
655
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
656
access = _KnitKeyAccess(transport, mapper)
657
return KnitVersionedFiles(index, access, annotated=annotated)
661
def make_pack_factory(graph, delta, keylength):
662
"""Create a factory for creating a pack based VersionedFiles.
664
This is only functional enough to run interface tests, it doesn't try to
665
provide a full pack environment.
667
:param graph: Store a graph.
668
:param delta: Delta compress contents.
669
:param keylength: How long should keys be.
671
def factory(transport):
672
parents = graph or delta
678
max_delta_chain = 200
681
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
682
key_elements=keylength)
683
stream = transport.open_write_stream('newpack')
684
writer = pack.ContainerWriter(stream.write)
686
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
687
deltas=delta, add_callback=graph_index.add_nodes)
688
access = _DirectPackAccess({})
689
access.set_writer(writer, graph_index, (transport, 'newpack'))
690
result = KnitVersionedFiles(index, access,
691
max_delta_chain=max_delta_chain)
692
result.stream = stream
693
result.writer = writer
698
def cleanup_pack_knit(versioned_files):
699
versioned_files.stream.close()
700
versioned_files.writer.end()
703
class KnitVersionedFiles(VersionedFiles):
704
"""Storage for many versioned files using knit compression.
706
Backend storage is managed by indices and data objects.
708
:ivar _index: A _KnitGraphIndex or similar that can describe the
709
parents, graph, compression and data location of entries in this
710
KnitVersionedFiles. Note that this is only the index for
711
*this* vfs; if there are fallbacks they must be queried separately.
714
def __init__(self, index, data_access, max_delta_chain=200,
715
annotated=False, reload_func=None):
716
"""Create a KnitVersionedFiles with index and data_access.
718
:param index: The index for the knit data.
719
:param data_access: The access object to store and retrieve knit
721
:param max_delta_chain: The maximum number of deltas to permit during
722
insertion. Set to 0 to prohibit the use of deltas.
723
:param annotated: Set to True to cause annotations to be calculated and
724
stored during insertion.
725
:param reload_func: An function that can be called if we think we need
726
to reload the pack listing and try again. See
727
'bzrlib.repofmt.pack_repo.AggregateIndex' for the signature.
730
self._access = data_access
731
self._max_delta_chain = max_delta_chain
733
self._factory = KnitAnnotateFactory()
735
self._factory = KnitPlainFactory()
736
self._fallback_vfs = []
737
self._reload_func = reload_func
740
return "%s(%r, %r)" % (
741
self.__class__.__name__,
745
def add_fallback_versioned_files(self, a_versioned_files):
746
"""Add a source of texts for texts not present in this knit.
748
:param a_versioned_files: A VersionedFiles object.
750
self._fallback_vfs.append(a_versioned_files)
752
def add_lines(self, key, parents, lines, parent_texts=None,
753
left_matching_blocks=None, nostore_sha=None, random_id=False,
755
"""See VersionedFiles.add_lines()."""
756
self._index._check_write_ok()
757
self._check_add(key, lines, random_id, check_content)
759
# The caller might pass None if there is no graph data, but kndx
760
# indexes can't directly store that, so we give them
761
# an empty tuple instead.
763
return self._add(key, lines, parents,
764
parent_texts, left_matching_blocks, nostore_sha, random_id)
766
def _add(self, key, lines, parents, parent_texts,
767
left_matching_blocks, nostore_sha, random_id):
768
"""Add a set of lines on top of version specified by parents.
770
Any versions not present will be converted into ghosts.
772
# first thing, if the content is something we don't need to store, find
774
line_bytes = ''.join(lines)
775
digest = sha_string(line_bytes)
776
if nostore_sha == digest:
777
raise errors.ExistingContent
780
if parent_texts is None:
782
# Do a single query to ascertain parent presence; we only compress
783
# against parents in the same kvf.
784
present_parent_map = self._index.get_parent_map(parents)
785
for parent in parents:
786
if parent in present_parent_map:
787
present_parents.append(parent)
789
# Currently we can only compress against the left most present parent.
790
if (len(present_parents) == 0 or
791
present_parents[0] != parents[0]):
794
# To speed the extract of texts the delta chain is limited
795
# to a fixed number of deltas. This should minimize both
796
# I/O and the time spend applying deltas.
797
delta = self._check_should_delta(present_parents[0])
799
text_length = len(line_bytes)
802
if lines[-1][-1] != '\n':
803
# copy the contents of lines.
805
options.append('no-eol')
806
lines[-1] = lines[-1] + '\n'
810
if type(element) != str:
811
raise TypeError("key contains non-strings: %r" % (key,))
812
# Knit hunks are still last-element only
814
content = self._factory.make(lines, version_id)
815
if 'no-eol' in options:
816
# Hint to the content object that its text() call should strip the
818
content._should_strip_eol = True
819
if delta or (self._factory.annotated and len(present_parents) > 0):
820
# Merge annotations from parent texts if needed.
821
delta_hunks = self._merge_annotations(content, present_parents,
822
parent_texts, delta, self._factory.annotated,
823
left_matching_blocks)
826
options.append('line-delta')
827
store_lines = self._factory.lower_line_delta(delta_hunks)
828
size, bytes = self._record_to_data(key, digest,
831
options.append('fulltext')
832
# isinstance is slower and we have no hierarchy.
833
if self._factory.__class__ == KnitPlainFactory:
834
# Use the already joined bytes saving iteration time in
836
size, bytes = self._record_to_data(key, digest,
839
# get mixed annotation + content and feed it into the
841
store_lines = self._factory.lower_fulltext(content)
842
size, bytes = self._record_to_data(key, digest,
845
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
846
self._index.add_records(
847
((key, options, access_memo, parents),),
849
return digest, text_length, content
851
def annotate(self, key):
852
"""See VersionedFiles.annotate."""
853
return self._factory.annotate(self, key)
855
def check(self, progress_bar=None):
856
"""See VersionedFiles.check()."""
857
# This doesn't actually test extraction of everything, but that will
858
# impact 'bzr check' substantially, and needs to be integrated with
859
# care. However, it does check for the obvious problem of a delta with
861
keys = self._index.keys()
862
parent_map = self.get_parent_map(keys)
864
if self._index.get_method(key) != 'fulltext':
865
compression_parent = parent_map[key][0]
866
if compression_parent not in parent_map:
867
raise errors.KnitCorrupt(self,
868
"Missing basis parent %s for %s" % (
869
compression_parent, key))
870
for fallback_vfs in self._fallback_vfs:
873
def _check_add(self, key, lines, random_id, check_content):
874
"""check that version_id and lines are safe to add."""
876
if contains_whitespace(version_id):
877
raise InvalidRevisionId(version_id, self)
878
self.check_not_reserved_id(version_id)
879
# TODO: If random_id==False and the key is already present, we should
880
# probably check that the existing content is identical to what is
881
# being inserted, and otherwise raise an exception. This would make
882
# the bundle code simpler.
884
self._check_lines_not_unicode(lines)
885
self._check_lines_are_lines(lines)
887
def _check_header(self, key, line):
888
rec = self._split_header(line)
889
self._check_header_version(rec, key[-1])
892
def _check_header_version(self, rec, version_id):
893
"""Checks the header version on original format knit records.
895
These have the last component of the key embedded in the record.
897
if rec[1] != version_id:
898
raise KnitCorrupt(self,
899
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
901
def _check_should_delta(self, parent):
902
"""Iterate back through the parent listing, looking for a fulltext.
904
This is used when we want to decide whether to add a delta or a new
905
fulltext. It searches for _max_delta_chain parents. When it finds a
906
fulltext parent, it sees if the total size of the deltas leading up to
907
it is large enough to indicate that we want a new full text anyway.
909
Return True if we should create a new delta, False if we should use a
914
for count in xrange(self._max_delta_chain):
916
# Note that this only looks in the index of this particular
917
# KnitVersionedFiles, not in the fallbacks. This ensures that
918
# we won't store a delta spanning physical repository
920
build_details = self._index.get_build_details([parent])
921
parent_details = build_details[parent]
922
except (RevisionNotPresent, KeyError), e:
923
# Some basis is not locally present: always fulltext
925
index_memo, compression_parent, _, _ = parent_details
926
_, _, size = index_memo
927
if compression_parent is None:
931
# We don't explicitly check for presence because this is in an
932
# inner loop, and if it's missing it'll fail anyhow.
933
parent = compression_parent
935
# We couldn't find a fulltext, so we must create a new one
937
# Simple heuristic - if the total I/O wold be greater as a delta than
938
# the originally installed fulltext, we create a new fulltext.
939
return fulltext_size > delta_size
941
def _build_details_to_components(self, build_details):
942
"""Convert a build_details tuple to a position tuple."""
943
# record_details, access_memo, compression_parent
944
return build_details[3], build_details[0], build_details[1]
946
def _get_components_positions(self, keys, allow_missing=False):
947
"""Produce a map of position data for the components of keys.
949
This data is intended to be used for retrieving the knit records.
951
A dict of key to (record_details, index_memo, next, parents) is
953
method is the way referenced data should be applied.
954
index_memo is the handle to pass to the data access to actually get the
956
next is the build-parent of the version, or None for fulltexts.
957
parents is the version_ids of the parents of this version
959
:param allow_missing: If True do not raise an error on a missing component,
963
pending_components = keys
964
while pending_components:
965
build_details = self._index.get_build_details(pending_components)
966
current_components = set(pending_components)
967
pending_components = set()
968
for key, details in build_details.iteritems():
969
(index_memo, compression_parent, parents,
970
record_details) = details
971
method = record_details[0]
972
if compression_parent is not None:
973
pending_components.add(compression_parent)
974
component_data[key] = self._build_details_to_components(details)
975
missing = current_components.difference(build_details)
976
if missing and not allow_missing:
977
raise errors.RevisionNotPresent(missing.pop(), self)
978
return component_data
980
def _get_content(self, key, parent_texts={}):
981
"""Returns a content object that makes up the specified
983
cached_version = parent_texts.get(key, None)
984
if cached_version is not None:
985
# Ensure the cache dict is valid.
986
if not self.get_parent_map([key]):
987
raise RevisionNotPresent(key, self)
988
return cached_version
989
text_map, contents_map = self._get_content_maps([key])
990
return contents_map[key]
992
def _get_content_maps(self, keys, nonlocal_keys=None):
993
"""Produce maps of text and KnitContents
995
:param keys: The keys to produce content maps for.
996
:param nonlocal_keys: An iterable of keys(possibly intersecting keys)
997
which are known to not be in this knit, but rather in one of the
999
:return: (text_map, content_map) where text_map contains the texts for
1000
the requested versions and content_map contains the KnitContents.
1002
# FUTURE: This function could be improved for the 'extract many' case
1003
# by tracking each component and only doing the copy when the number of
1004
# children than need to apply delta's to it is > 1 or it is part of the
1007
multiple_versions = len(keys) != 1
1008
record_map = self._get_record_map(keys, allow_missing=True)
1013
if nonlocal_keys is None:
1014
nonlocal_keys = set()
1016
nonlocal_keys = frozenset(nonlocal_keys)
1017
missing_keys = set(nonlocal_keys)
1018
for source in self._fallback_vfs:
1019
if not missing_keys:
1021
for record in source.get_record_stream(missing_keys,
1023
if record.storage_kind == 'absent':
1025
missing_keys.remove(record.key)
1026
lines = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
1027
text_map[record.key] = lines
1028
content_map[record.key] = PlainKnitContent(lines, record.key)
1029
if record.key in keys:
1030
final_content[record.key] = content_map[record.key]
1032
if key in nonlocal_keys:
1037
while cursor is not None:
1039
record, record_details, digest, next = record_map[cursor]
1041
raise RevisionNotPresent(cursor, self)
1042
components.append((cursor, record, record_details, digest))
1044
if cursor in content_map:
1045
# no need to plan further back
1046
components.append((cursor, None, None, None))
1050
for (component_id, record, record_details,
1051
digest) in reversed(components):
1052
if component_id in content_map:
1053
content = content_map[component_id]
1055
content, delta = self._factory.parse_record(key[-1],
1056
record, record_details, content,
1057
copy_base_content=multiple_versions)
1058
if multiple_versions:
1059
content_map[component_id] = content
1061
final_content[key] = content
1063
# digest here is the digest from the last applied component.
1064
text = content.text()
1065
actual_sha = sha_strings(text)
1066
if actual_sha != digest:
1067
raise SHA1KnitCorrupt(self, actual_sha, digest, key, text)
1068
text_map[key] = text
1069
return text_map, final_content
1071
def get_parent_map(self, keys):
1072
"""Get a map of the graph parents of keys.
1074
:param keys: The keys to look up parents for.
1075
:return: A mapping from keys to parents. Absent keys are absent from
1078
return self._get_parent_map_with_sources(keys)[0]
1080
def _get_parent_map_with_sources(self, keys):
1081
"""Get a map of the parents of keys.
1083
:param keys: The keys to look up parents for.
1084
:return: A tuple. The first element is a mapping from keys to parents.
1085
Absent keys are absent from the mapping. The second element is a
1086
list with the locations each key was found in. The first element
1087
is the in-this-knit parents, the second the first fallback source,
1091
sources = [self._index] + self._fallback_vfs
1094
for source in sources:
1097
new_result = source.get_parent_map(missing)
1098
source_results.append(new_result)
1099
result.update(new_result)
1100
missing.difference_update(set(new_result))
1101
return result, source_results
1103
def _get_record_map(self, keys, allow_missing=False):
1104
"""Produce a dictionary of knit records.
1106
:return: {key:(record, record_details, digest, next)}
1108
data returned from read_records
1110
opaque information to pass to parse_record
1112
SHA1 digest of the full text after all steps are done
1114
build-parent of the version, i.e. the leftmost ancestor.
1115
Will be None if the record is not a delta.
1116
:param keys: The keys to build a map for
1117
:param allow_missing: If some records are missing, rather than
1118
error, just return the data that could be generated.
1120
# This retries the whole request if anything fails. Potentially we
1121
# could be a bit more selective. We could track the keys whose records
1122
# we have successfully found, and then only request the new records
1123
# from there. However, _get_components_positions grabs the whole build
1124
# chain, which means we'll likely try to grab the same records again
1125
# anyway. Also, can the build chains change as part of a pack
1126
# operation? We wouldn't want to end up with a broken chain.
1129
position_map = self._get_components_positions(keys,
1130
allow_missing=allow_missing)
1131
# key = component_id, r = record_details, i_m = index_memo,
1133
records = [(key, i_m) for key, (r, i_m, n)
1134
in position_map.iteritems()]
1136
for key, record, digest in self._read_records_iter(records):
1137
(record_details, index_memo, next) = position_map[key]
1138
record_map[key] = record, record_details, digest, next
1140
except errors.RetryWithNewPacks, e:
1141
self._access.reload_or_raise(e)
1143
def _split_by_prefix(self, keys):
1144
"""For the given keys, split them up based on their prefix.
1146
To keep memory pressure somewhat under control, split the
1147
requests back into per-file-id requests, otherwise "bzr co"
1148
extracts the full tree into memory before writing it to disk.
1149
This should be revisited if _get_content_maps() can ever cross
1152
:param keys: An iterable of key tuples
1153
:return: A dict of {prefix: [key_list]}
1155
split_by_prefix = {}
1158
split_by_prefix.setdefault('', []).append(key)
1160
split_by_prefix.setdefault(key[0], []).append(key)
1161
return split_by_prefix
1163
def get_record_stream(self, keys, ordering, include_delta_closure):
1164
"""Get a stream of records for keys.
1166
:param keys: The keys to include.
1167
:param ordering: Either 'unordered' or 'topological'. A topologically
1168
sorted stream has compression parents strictly before their
1170
:param include_delta_closure: If True then the closure across any
1171
compression parents will be included (in the opaque data).
1172
:return: An iterator of ContentFactory objects, each of which is only
1173
valid until the iterator is advanced.
1175
# keys might be a generator
1179
if not self._index.has_graph:
1180
# Cannot topological order when no graph has been stored.
1181
ordering = 'unordered'
1183
remaining_keys = keys
1186
keys = set(remaining_keys)
1187
for content_factory in self._get_remaining_record_stream(keys,
1188
ordering, include_delta_closure):
1189
remaining_keys.discard(content_factory.key)
1190
yield content_factory
1192
except errors.RetryWithNewPacks, e:
1193
self._access.reload_or_raise(e)
1195
def _get_remaining_record_stream(self, keys, ordering,
1196
include_delta_closure):
1197
"""This function is the 'retry' portion for get_record_stream."""
1198
if include_delta_closure:
1199
positions = self._get_components_positions(keys, allow_missing=True)
1201
build_details = self._index.get_build_details(keys)
1203
# (record_details, access_memo, compression_parent_key)
1204
positions = dict((key, self._build_details_to_components(details))
1205
for key, details in build_details.iteritems())
1206
absent_keys = keys.difference(set(positions))
1207
# There may be more absent keys : if we're missing the basis component
1208
# and are trying to include the delta closure.
1209
if include_delta_closure:
1210
needed_from_fallback = set()
1211
# Build up reconstructable_keys dict. key:True in this dict means
1212
# the key can be reconstructed.
1213
reconstructable_keys = {}
1217
chain = [key, positions[key][2]]
1219
needed_from_fallback.add(key)
1222
while chain[-1] is not None:
1223
if chain[-1] in reconstructable_keys:
1224
result = reconstructable_keys[chain[-1]]
1228
chain.append(positions[chain[-1]][2])
1230
# missing basis component
1231
needed_from_fallback.add(chain[-1])
1234
for chain_key in chain[:-1]:
1235
reconstructable_keys[chain_key] = result
1237
needed_from_fallback.add(key)
1238
# Double index lookups here : need a unified api ?
1239
global_map, parent_maps = self._get_parent_map_with_sources(keys)
1240
if ordering == 'topological':
1241
# Global topological sort
1242
present_keys = tsort.topo_sort(global_map)
1243
# Now group by source:
1245
current_source = None
1246
for key in present_keys:
1247
for parent_map in parent_maps:
1248
if key in parent_map:
1249
key_source = parent_map
1251
if current_source is not key_source:
1252
source_keys.append((key_source, []))
1253
current_source = key_source
1254
source_keys[-1][1].append(key)
1256
if ordering != 'unordered':
1257
raise AssertionError('valid values for ordering are:'
1258
' "unordered" or "topological" not: %r'
1260
# Just group by source; remote sources first.
1263
for parent_map in reversed(parent_maps):
1264
source_keys.append((parent_map, []))
1265
for key in parent_map:
1266
present_keys.append(key)
1267
source_keys[-1][1].append(key)
1268
# We have been requested to return these records in an order that
1269
# suits us. So we ask the index to give us an optimally sorted
1271
for source, sub_keys in source_keys:
1272
if source is parent_maps[0]:
1273
# Only sort the keys for this VF
1274
self._index._sort_keys_by_io(sub_keys, positions)
1275
absent_keys = keys - set(global_map)
1276
for key in absent_keys:
1277
yield AbsentContentFactory(key)
1278
# restrict our view to the keys we can answer.
1279
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1280
# XXX: At that point we need to consider the impact of double reads by
1281
# utilising components multiple times.
1282
if include_delta_closure:
1283
# XXX: get_content_maps performs its own index queries; allow state
1285
non_local_keys = needed_from_fallback - absent_keys
1286
prefix_split_keys = self._split_by_prefix(present_keys)
1287
prefix_split_non_local_keys = self._split_by_prefix(non_local_keys)
1288
for prefix, keys in prefix_split_keys.iteritems():
1289
non_local = prefix_split_non_local_keys.get(prefix, [])
1290
non_local = set(non_local)
1291
text_map, _ = self._get_content_maps(keys, non_local)
1293
lines = text_map.pop(key)
1294
yield ChunkedContentFactory(key, global_map[key], None,
1297
for source, keys in source_keys:
1298
if source is parent_maps[0]:
1299
# this KnitVersionedFiles
1300
records = [(key, positions[key][1]) for key in keys]
1301
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1302
(record_details, index_memo, _) = positions[key]
1303
yield KnitContentFactory(key, global_map[key],
1304
record_details, sha1, raw_data, self._factory.annotated, None)
1306
vf = self._fallback_vfs[parent_maps.index(source) - 1]
1307
for record in vf.get_record_stream(keys, ordering,
1308
include_delta_closure):
1311
def get_sha1s(self, keys):
1312
"""See VersionedFiles.get_sha1s()."""
1314
record_map = self._get_record_map(missing, allow_missing=True)
1316
for key, details in record_map.iteritems():
1317
if key not in missing:
1319
# record entry 2 is the 'digest'.
1320
result[key] = details[2]
1321
missing.difference_update(set(result))
1322
for source in self._fallback_vfs:
1325
new_result = source.get_sha1s(missing)
1326
result.update(new_result)
1327
missing.difference_update(set(new_result))
1330
def insert_record_stream(self, stream):
1331
"""Insert a record stream into this container.
1333
:param stream: A stream of records to insert.
1335
:seealso VersionedFiles.get_record_stream:
1337
def get_adapter(adapter_key):
1339
return adapters[adapter_key]
1341
adapter_factory = adapter_registry.get(adapter_key)
1342
adapter = adapter_factory(self)
1343
adapters[adapter_key] = adapter
1346
if self._factory.annotated:
1347
# self is annotated, we need annotated knits to use directly.
1348
annotated = "annotated-"
1351
# self is not annotated, but we can strip annotations cheaply.
1353
convertibles = set(["knit-annotated-ft-gz"])
1354
if self._max_delta_chain:
1355
delta_types.add("knit-annotated-delta-gz")
1356
convertibles.add("knit-annotated-delta-gz")
1357
# The set of types we can cheaply adapt without needing basis texts.
1358
native_types = set()
1359
if self._max_delta_chain:
1360
native_types.add("knit-%sdelta-gz" % annotated)
1361
delta_types.add("knit-%sdelta-gz" % annotated)
1362
native_types.add("knit-%sft-gz" % annotated)
1363
knit_types = native_types.union(convertibles)
1365
# Buffer all index entries that we can't add immediately because their
1366
# basis parent is missing. We don't buffer all because generating
1367
# annotations may require access to some of the new records. However we
1368
# can't generate annotations from new deltas until their basis parent
1369
# is present anyway, so we get away with not needing an index that
1370
# includes the new keys.
1372
# See <http://launchpad.net/bugs/300177> about ordering of compression
1373
# parents in the records - to be conservative, we insist that all
1374
# parents must be present to avoid expanding to a fulltext.
1376
# key = basis_parent, value = index entry to add
1377
buffered_index_entries = {}
1378
for record in stream:
1379
parents = record.parents
1380
if record.storage_kind in delta_types:
1381
# TODO: eventually the record itself should track
1382
# compression_parent
1383
compression_parent = parents[0]
1385
compression_parent = None
1386
# Raise an error when a record is missing.
1387
if record.storage_kind == 'absent':
1388
raise RevisionNotPresent([record.key], self)
1389
elif ((record.storage_kind in knit_types)
1390
and (compression_parent is None
1391
or not self._fallback_vfs
1392
or self._index.has_key(compression_parent)
1393
or not self.has_key(compression_parent))):
1394
# we can insert the knit record literally if either it has no
1395
# compression parent OR we already have its basis in this kvf
1396
# OR the basis is not present even in the fallbacks. In the
1397
# last case it will either turn up later in the stream and all
1398
# will be well, or it won't turn up at all and we'll raise an
1401
# TODO: self.has_key is somewhat redundant with
1402
# self._index.has_key; we really want something that directly
1403
# asks if it's only present in the fallbacks. -- mbp 20081119
1404
if record.storage_kind not in native_types:
1406
adapter_key = (record.storage_kind, "knit-delta-gz")
1407
adapter = get_adapter(adapter_key)
1409
adapter_key = (record.storage_kind, "knit-ft-gz")
1410
adapter = get_adapter(adapter_key)
1411
bytes = adapter.get_bytes(
1412
record, record.get_bytes_as(record.storage_kind))
1414
bytes = record.get_bytes_as(record.storage_kind)
1415
options = [record._build_details[0]]
1416
if record._build_details[1]:
1417
options.append('no-eol')
1418
# Just blat it across.
1419
# Note: This does end up adding data on duplicate keys. As
1420
# modern repositories use atomic insertions this should not
1421
# lead to excessive growth in the event of interrupted fetches.
1422
# 'knit' repositories may suffer excessive growth, but as a
1423
# deprecated format this is tolerable. It can be fixed if
1424
# needed by in the kndx index support raising on a duplicate
1425
# add with identical parents and options.
1426
access_memo = self._access.add_raw_records(
1427
[(record.key, len(bytes))], bytes)[0]
1428
index_entry = (record.key, options, access_memo, parents)
1430
if 'fulltext' not in options:
1431
# Not a fulltext, so we need to make sure the compression
1432
# parent will also be present.
1433
# Note that pack backed knits don't need to buffer here
1434
# because they buffer all writes to the transaction level,
1435
# but we don't expose that difference at the index level. If
1436
# the query here has sufficient cost to show up in
1437
# profiling we should do that.
1439
# They're required to be physically in this
1440
# KnitVersionedFiles, not in a fallback.
1441
if not self._index.has_key(compression_parent):
1442
pending = buffered_index_entries.setdefault(
1443
compression_parent, [])
1444
pending.append(index_entry)
1447
self._index.add_records([index_entry])
1448
elif record.storage_kind == 'chunked':
1449
self.add_lines(record.key, parents,
1450
osutils.chunks_to_lines(record.get_bytes_as('chunked')))
1451
elif record.storage_kind == 'fulltext':
1452
self.add_lines(record.key, parents,
1453
split_lines(record.get_bytes_as('fulltext')))
1455
# Not a fulltext, and not suitable for direct insertion as a
1456
# delta, either because it's not the right format, or this
1457
# KnitVersionedFiles doesn't permit deltas (_max_delta_chain ==
1458
# 0) or because it depends on a base only present in the
1460
adapter_key = record.storage_kind, 'fulltext'
1461
adapter = get_adapter(adapter_key)
1462
lines = split_lines(adapter.get_bytes(
1463
record, record.get_bytes_as(record.storage_kind)))
1465
self.add_lines(record.key, parents, lines)
1466
except errors.RevisionAlreadyPresent:
1468
# Add any records whose basis parent is now available.
1469
added_keys = [record.key]
1471
key = added_keys.pop(0)
1472
if key in buffered_index_entries:
1473
index_entries = buffered_index_entries[key]
1474
self._index.add_records(index_entries)
1476
[index_entry[0] for index_entry in index_entries])
1477
del buffered_index_entries[key]
1478
# If there were any deltas which had a missing basis parent, error.
1479
if buffered_index_entries:
1480
from pprint import pformat
1481
raise errors.BzrCheckError(
1482
"record_stream refers to compression parents not in %r:\n%s"
1483
% (self, pformat(sorted(buffered_index_entries.keys()))))
1485
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1486
"""Iterate over the lines in the versioned files from keys.
1488
This may return lines from other keys. Each item the returned
1489
iterator yields is a tuple of a line and a text version that that line
1490
is present in (not introduced in).
1492
Ordering of results is in whatever order is most suitable for the
1493
underlying storage format.
1495
If a progress bar is supplied, it may be used to indicate progress.
1496
The caller is responsible for cleaning up progress bars (because this
1500
* Lines are normalised by the underlying store: they will all have \\n
1502
* Lines are returned in arbitrary order.
1503
* If a requested key did not change any lines (or didn't have any
1504
lines), it may not be mentioned at all in the result.
1506
:return: An iterator over (line, key).
1509
pb = progress.DummyProgress()
1515
# we don't care about inclusions, the caller cares.
1516
# but we need to setup a list of records to visit.
1517
# we need key, position, length
1519
build_details = self._index.get_build_details(keys)
1520
for key, details in build_details.iteritems():
1522
key_records.append((key, details[0]))
1523
records_iter = enumerate(self._read_records_iter(key_records))
1524
for (key_idx, (key, data, sha_value)) in records_iter:
1525
pb.update('Walking content.', key_idx, total)
1526
compression_parent = build_details[key][1]
1527
if compression_parent is None:
1529
line_iterator = self._factory.get_fulltext_content(data)
1532
line_iterator = self._factory.get_linedelta_content(data)
1533
# Now that we are yielding the data for this key, remove it
1536
# XXX: It might be more efficient to yield (key,
1537
# line_iterator) in the future. However for now, this is a
1538
# simpler change to integrate into the rest of the
1539
# codebase. RBC 20071110
1540
for line in line_iterator:
1543
except errors.RetryWithNewPacks, e:
1544
self._access.reload_or_raise(e)
1545
# If there are still keys we've not yet found, we look in the fallback
1546
# vfs, and hope to find them there. Note that if the keys are found
1547
# but had no changes or no content, the fallback may not return
1549
if keys and not self._fallback_vfs:
1550
# XXX: strictly the second parameter is meant to be the file id
1551
# but it's not easily accessible here.
1552
raise RevisionNotPresent(keys, repr(self))
1553
for source in self._fallback_vfs:
1557
for line, key in source.iter_lines_added_or_present_in_keys(keys):
1558
source_keys.add(key)
1560
keys.difference_update(source_keys)
1561
pb.update('Walking content.', total, total)
1563
def _make_line_delta(self, delta_seq, new_content):
1564
"""Generate a line delta from delta_seq and new_content."""
1566
for op in delta_seq.get_opcodes():
1567
if op[0] == 'equal':
1569
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1572
def _merge_annotations(self, content, parents, parent_texts={},
1573
delta=None, annotated=None,
1574
left_matching_blocks=None):
1575
"""Merge annotations for content and generate deltas.
1577
This is done by comparing the annotations based on changes to the text
1578
and generating a delta on the resulting full texts. If annotations are
1579
not being created then a simple delta is created.
1581
if left_matching_blocks is not None:
1582
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1586
for parent_key in parents:
1587
merge_content = self._get_content(parent_key, parent_texts)
1588
if (parent_key == parents[0] and delta_seq is not None):
1591
seq = patiencediff.PatienceSequenceMatcher(
1592
None, merge_content.text(), content.text())
1593
for i, j, n in seq.get_matching_blocks():
1596
# this copies (origin, text) pairs across to the new
1597
# content for any line that matches the last-checked
1599
content._lines[j:j+n] = merge_content._lines[i:i+n]
1600
# XXX: Robert says the following block is a workaround for a
1601
# now-fixed bug and it can probably be deleted. -- mbp 20080618
1602
if content._lines and content._lines[-1][1][-1] != '\n':
1603
# The copied annotation was from a line without a trailing EOL,
1604
# reinstate one for the content object, to ensure correct
1606
line = content._lines[-1][1] + '\n'
1607
content._lines[-1] = (content._lines[-1][0], line)
1609
if delta_seq is None:
1610
reference_content = self._get_content(parents[0], parent_texts)
1611
new_texts = content.text()
1612
old_texts = reference_content.text()
1613
delta_seq = patiencediff.PatienceSequenceMatcher(
1614
None, old_texts, new_texts)
1615
return self._make_line_delta(delta_seq, content)
1617
def _parse_record(self, version_id, data):
1618
"""Parse an original format knit record.
1620
These have the last element of the key only present in the stored data.
1622
rec, record_contents = self._parse_record_unchecked(data)
1623
self._check_header_version(rec, version_id)
1624
return record_contents, rec[3]
1626
def _parse_record_header(self, key, raw_data):
1627
"""Parse a record header for consistency.
1629
:return: the header and the decompressor stream.
1630
as (stream, header_record)
1632
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(raw_data))
1635
rec = self._check_header(key, df.readline())
1636
except Exception, e:
1637
raise KnitCorrupt(self,
1638
"While reading {%s} got %s(%s)"
1639
% (key, e.__class__.__name__, str(e)))
1642
def _parse_record_unchecked(self, data):
1644
# 4168 calls in 2880 217 internal
1645
# 4168 calls to _parse_record_header in 2121
1646
# 4168 calls to readlines in 330
1647
df = tuned_gzip.GzipFile(mode='rb', fileobj=StringIO(data))
1649
record_contents = df.readlines()
1650
except Exception, e:
1651
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1652
(data, e.__class__.__name__, str(e)))
1653
header = record_contents.pop(0)
1654
rec = self._split_header(header)
1655
last_line = record_contents.pop()
1656
if len(record_contents) != int(rec[2]):
1657
raise KnitCorrupt(self,
1658
'incorrect number of lines %s != %s'
1659
' for version {%s} %s'
1660
% (len(record_contents), int(rec[2]),
1661
rec[1], record_contents))
1662
if last_line != 'end %s\n' % rec[1]:
1663
raise KnitCorrupt(self,
1664
'unexpected version end line %r, wanted %r'
1665
% (last_line, rec[1]))
1667
return rec, record_contents
1669
def _read_records_iter(self, records):
1670
"""Read text records from data file and yield result.
1672
The result will be returned in whatever is the fastest to read.
1673
Not by the order requested. Also, multiple requests for the same
1674
record will only yield 1 response.
1675
:param records: A list of (key, access_memo) entries
1676
:return: Yields (key, contents, digest) in the order
1677
read, not the order requested
1682
# XXX: This smells wrong, IO may not be getting ordered right.
1683
needed_records = sorted(set(records), key=operator.itemgetter(1))
1684
if not needed_records:
1687
# The transport optimizes the fetching as well
1688
# (ie, reads continuous ranges.)
1689
raw_data = self._access.get_raw_records(
1690
[index_memo for key, index_memo in needed_records])
1692
for (key, index_memo), data in \
1693
izip(iter(needed_records), raw_data):
1694
content, digest = self._parse_record(key[-1], data)
1695
yield key, content, digest
1697
def _read_records_iter_raw(self, records):
1698
"""Read text records from data file and yield raw data.
1700
This unpacks enough of the text record to validate the id is
1701
as expected but thats all.
1703
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1705
# setup an iterator of the external records:
1706
# uses readv so nice and fast we hope.
1708
# grab the disk data needed.
1709
needed_offsets = [index_memo for key, index_memo
1711
raw_records = self._access.get_raw_records(needed_offsets)
1713
for key, index_memo in records:
1714
data = raw_records.next()
1715
# validate the header (note that we can only use the suffix in
1716
# current knit records).
1717
df, rec = self._parse_record_header(key, data)
1719
yield key, data, rec[3]
1721
def _record_to_data(self, key, digest, lines, dense_lines=None):
1722
"""Convert key, digest, lines into a raw data block.
1724
:param key: The key of the record. Currently keys are always serialised
1725
using just the trailing component.
1726
:param dense_lines: The bytes of lines but in a denser form. For
1727
instance, if lines is a list of 1000 bytestrings each ending in \n,
1728
dense_lines may be a list with one line in it, containing all the
1729
1000's lines and their \n's. Using dense_lines if it is already
1730
known is a win because the string join to create bytes in this
1731
function spends less time resizing the final string.
1732
:return: (len, a StringIO instance with the raw data ready to read.)
1734
# Note: using a string copy here increases memory pressure with e.g.
1735
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1736
# when doing the initial commit of a mozilla tree. RBC 20070921
1737
bytes = ''.join(chain(
1738
["version %s %d %s\n" % (key[-1],
1741
dense_lines or lines,
1742
["end %s\n" % key[-1]]))
1743
if type(bytes) != str:
1744
raise AssertionError(
1745
'data must be plain bytes was %s' % type(bytes))
1746
if lines and lines[-1][-1] != '\n':
1747
raise ValueError('corrupt lines value %r' % lines)
1748
compressed_bytes = tuned_gzip.bytes_to_gzip(bytes)
1749
return len(compressed_bytes), compressed_bytes
1751
def _split_header(self, line):
1754
raise KnitCorrupt(self,
1755
'unexpected number of elements in record header')
1759
"""See VersionedFiles.keys."""
1760
if 'evil' in debug.debug_flags:
1761
trace.mutter_callsite(2, "keys scales with size of history")
1762
sources = [self._index] + self._fallback_vfs
1764
for source in sources:
1765
result.update(source.keys())
1769
class _KndxIndex(object):
1770
"""Manages knit index files
1772
The index is kept in memory and read on startup, to enable
1773
fast lookups of revision information. The cursor of the index
1774
file is always pointing to the end, making it easy to append
1777
_cache is a cache for fast mapping from version id to a Index
1780
_history is a cache for fast mapping from indexes to version ids.
1782
The index data format is dictionary compressed when it comes to
1783
parent references; a index entry may only have parents that with a
1784
lover index number. As a result, the index is topological sorted.
1786
Duplicate entries may be written to the index for a single version id
1787
if this is done then the latter one completely replaces the former:
1788
this allows updates to correct version and parent information.
1789
Note that the two entries may share the delta, and that successive
1790
annotations and references MUST point to the first entry.
1792
The index file on disc contains a header, followed by one line per knit
1793
record. The same revision can be present in an index file more than once.
1794
The first occurrence gets assigned a sequence number starting from 0.
1796
The format of a single line is
1797
REVISION_ID FLAGS BYTE_OFFSET LENGTH( PARENT_ID|PARENT_SEQUENCE_ID)* :\n
1798
REVISION_ID is a utf8-encoded revision id
1799
FLAGS is a comma separated list of flags about the record. Values include
1800
no-eol, line-delta, fulltext.
1801
BYTE_OFFSET is the ascii representation of the byte offset in the data file
1802
that the the compressed data starts at.
1803
LENGTH is the ascii representation of the length of the data file.
1804
PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
1806
PARENT_SEQUENCE_ID the ascii representation of the sequence number of a
1807
revision id already in the knit that is a parent of REVISION_ID.
1808
The ' :' marker is the end of record marker.
1811
when a write is interrupted to the index file, it will result in a line
1812
that does not end in ' :'. If the ' :' is not present at the end of a line,
1813
or at the end of the file, then the record that is missing it will be
1814
ignored by the parser.
1816
When writing new records to the index file, the data is preceded by '\n'
1817
to ensure that records always start on new lines even if the last write was
1818
interrupted. As a result its normal for the last line in the index to be
1819
missing a trailing newline. One can be added with no harmful effects.
1821
:ivar _kndx_cache: dict from prefix to the old state of KnitIndex objects,
1822
where prefix is e.g. the (fileid,) for .texts instances or () for
1823
constant-mapped things like .revisions, and the old state is
1824
tuple(history_vector, cache_dict). This is used to prevent having an
1825
ABI change with the C extension that reads .kndx files.
1828
HEADER = "# bzr knit index 8\n"
1830
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1831
"""Create a _KndxIndex on transport using mapper."""
1832
self._transport = transport
1833
self._mapper = mapper
1834
self._get_scope = get_scope
1835
self._allow_writes = allow_writes
1836
self._is_locked = is_locked
1838
self.has_graph = True
1840
def add_records(self, records, random_id=False):
1841
"""Add multiple records to the index.
1843
:param records: a list of tuples:
1844
(key, options, access_memo, parents).
1845
:param random_id: If True the ids being added were randomly generated
1846
and no check for existence will be performed.
1849
for record in records:
1852
path = self._mapper.map(key) + '.kndx'
1853
path_keys = paths.setdefault(path, (prefix, []))
1854
path_keys[1].append(record)
1855
for path in sorted(paths):
1856
prefix, path_keys = paths[path]
1857
self._load_prefixes([prefix])
1859
orig_history = self._kndx_cache[prefix][1][:]
1860
orig_cache = self._kndx_cache[prefix][0].copy()
1863
for key, options, (_, pos, size), parents in path_keys:
1865
# kndx indices cannot be parentless.
1867
line = "\n%s %s %s %s %s :" % (
1868
key[-1], ','.join(options), pos, size,
1869
self._dictionary_compress(parents))
1870
if type(line) != str:
1871
raise AssertionError(
1872
'data must be utf8 was %s' % type(line))
1874
self._cache_key(key, options, pos, size, parents)
1875
if len(orig_history):
1876
self._transport.append_bytes(path, ''.join(lines))
1878
self._init_index(path, lines)
1880
# If any problems happen, restore the original values and re-raise
1881
self._kndx_cache[prefix] = (orig_cache, orig_history)
1884
def _cache_key(self, key, options, pos, size, parent_keys):
1885
"""Cache a version record in the history array and index cache.
1887
This is inlined into _load_data for performance. KEEP IN SYNC.
1888
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1892
version_id = key[-1]
1893
# last-element only for compatibilty with the C load_data.
1894
parents = tuple(parent[-1] for parent in parent_keys)
1895
for parent in parent_keys:
1896
if parent[:-1] != prefix:
1897
raise ValueError("mismatched prefixes for %r, %r" % (
1899
cache, history = self._kndx_cache[prefix]
1900
# only want the _history index to reference the 1st index entry
1902
if version_id not in cache:
1903
index = len(history)
1904
history.append(version_id)
1906
index = cache[version_id][5]
1907
cache[version_id] = (version_id,
1914
def check_header(self, fp):
1915
line = fp.readline()
1917
# An empty file can actually be treated as though the file doesn't
1919
raise errors.NoSuchFile(self)
1920
if line != self.HEADER:
1921
raise KnitHeaderError(badline=line, filename=self)
1923
def _check_read(self):
1924
if not self._is_locked():
1925
raise errors.ObjectNotLocked(self)
1926
if self._get_scope() != self._scope:
1929
def _check_write_ok(self):
1930
"""Assert if not writes are permitted."""
1931
if not self._is_locked():
1932
raise errors.ObjectNotLocked(self)
1933
if self._get_scope() != self._scope:
1935
if self._mode != 'w':
1936
raise errors.ReadOnlyObjectDirtiedError(self)
1938
def get_build_details(self, keys):
1939
"""Get the method, index_memo and compression parent for keys.
1941
Ghosts are omitted from the result.
1943
:param keys: An iterable of keys.
1944
:return: A dict of key:(index_memo, compression_parent, parents,
1947
opaque structure to pass to read_records to extract the raw
1950
Content that this record is built upon, may be None
1952
Logical parents of this node
1954
extra information about the content which needs to be passed to
1955
Factory.parse_record
1957
parent_map = self.get_parent_map(keys)
1960
if key not in parent_map:
1962
method = self.get_method(key)
1963
parents = parent_map[key]
1964
if method == 'fulltext':
1965
compression_parent = None
1967
compression_parent = parents[0]
1968
noeol = 'no-eol' in self.get_options(key)
1969
index_memo = self.get_position(key)
1970
result[key] = (index_memo, compression_parent,
1971
parents, (method, noeol))
1974
def get_method(self, key):
1975
"""Return compression method of specified key."""
1976
options = self.get_options(key)
1977
if 'fulltext' in options:
1979
elif 'line-delta' in options:
1982
raise errors.KnitIndexUnknownMethod(self, options)
1984
def get_options(self, key):
1985
"""Return a list representing options.
1989
prefix, suffix = self._split_key(key)
1990
self._load_prefixes([prefix])
1992
return self._kndx_cache[prefix][0][suffix][1]
1994
raise RevisionNotPresent(key, self)
1996
def get_parent_map(self, keys):
1997
"""Get a map of the parents of keys.
1999
:param keys: The keys to look up parents for.
2000
:return: A mapping from keys to parents. Absent keys are absent from
2003
# Parse what we need to up front, this potentially trades off I/O
2004
# locality (.kndx and .knit in the same block group for the same file
2005
# id) for less checking in inner loops.
2006
prefixes = set(key[:-1] for key in keys)
2007
self._load_prefixes(prefixes)
2012
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
2016
result[key] = tuple(prefix + (suffix,) for
2017
suffix in suffix_parents)
2020
def get_position(self, key):
2021
"""Return details needed to access the version.
2023
:return: a tuple (key, data position, size) to hand to the access
2024
logic to get the record.
2026
prefix, suffix = self._split_key(key)
2027
self._load_prefixes([prefix])
2028
entry = self._kndx_cache[prefix][0][suffix]
2029
return key, entry[2], entry[3]
2031
has_key = _mod_index._has_key_from_parent_map
2033
def _init_index(self, path, extra_lines=[]):
2034
"""Initialize an index."""
2036
sio.write(self.HEADER)
2037
sio.writelines(extra_lines)
2039
self._transport.put_file_non_atomic(path, sio,
2040
create_parent_dir=True)
2041
# self._create_parent_dir)
2042
# mode=self._file_mode,
2043
# dir_mode=self._dir_mode)
2046
"""Get all the keys in the collection.
2048
The keys are not ordered.
2051
# Identify all key prefixes.
2052
# XXX: A bit hacky, needs polish.
2053
if type(self._mapper) == ConstantMapper:
2057
for quoted_relpath in self._transport.iter_files_recursive():
2058
path, ext = os.path.splitext(quoted_relpath)
2060
prefixes = [self._mapper.unmap(path) for path in relpaths]
2061
self._load_prefixes(prefixes)
2062
for prefix in prefixes:
2063
for suffix in self._kndx_cache[prefix][1]:
2064
result.add(prefix + (suffix,))
2067
def _load_prefixes(self, prefixes):
2068
"""Load the indices for prefixes."""
2070
for prefix in prefixes:
2071
if prefix not in self._kndx_cache:
2072
# the load_data interface writes to these variables.
2075
self._filename = prefix
2077
path = self._mapper.map(prefix) + '.kndx'
2078
fp = self._transport.get(path)
2080
# _load_data may raise NoSuchFile if the target knit is
2082
_load_data(self, fp)
2085
self._kndx_cache[prefix] = (self._cache, self._history)
2090
self._kndx_cache[prefix] = ({}, [])
2091
if type(self._mapper) == ConstantMapper:
2092
# preserve behaviour for revisions.kndx etc.
2093
self._init_index(path)
2098
missing_keys = _mod_index._missing_keys_from_parent_map
2100
def _partition_keys(self, keys):
2101
"""Turn keys into a dict of prefix:suffix_list."""
2104
prefix_keys = result.setdefault(key[:-1], [])
2105
prefix_keys.append(key[-1])
2108
def _dictionary_compress(self, keys):
2109
"""Dictionary compress keys.
2111
:param keys: The keys to generate references to.
2112
:return: A string representation of keys. keys which are present are
2113
dictionary compressed, and others are emitted as fulltext with a
2119
prefix = keys[0][:-1]
2120
cache = self._kndx_cache[prefix][0]
2122
if key[:-1] != prefix:
2123
# kndx indices cannot refer across partitioned storage.
2124
raise ValueError("mismatched prefixes for %r" % keys)
2125
if key[-1] in cache:
2126
# -- inlined lookup() --
2127
result_list.append(str(cache[key[-1]][5]))
2128
# -- end lookup () --
2130
result_list.append('.' + key[-1])
2131
return ' '.join(result_list)
2133
def _reset_cache(self):
2134
# Possibly this should be a LRU cache. A dictionary from key_prefix to
2135
# (cache_dict, history_vector) for parsed kndx files.
2136
self._kndx_cache = {}
2137
self._scope = self._get_scope()
2138
allow_writes = self._allow_writes()
2144
def _sort_keys_by_io(self, keys, positions):
2145
"""Figure out an optimal order to read the records for the given keys.
2147
Sort keys, grouped by index and sorted by position.
2149
:param keys: A list of keys whose records we want to read. This will be
2151
:param positions: A dict, such as the one returned by
2152
_get_components_positions()
2155
def get_sort_key(key):
2156
index_memo = positions[key][1]
2157
# Group by prefix and position. index_memo[0] is the key, so it is
2158
# (file_id, revision_id) and we don't want to sort on revision_id,
2159
# index_memo[1] is the position, and index_memo[2] is the size,
2160
# which doesn't matter for the sort
2161
return index_memo[0][:-1], index_memo[1]
2162
return keys.sort(key=get_sort_key)
2164
def _split_key(self, key):
2165
"""Split key into a prefix and suffix."""
2166
return key[:-1], key[-1]
2169
class _KnitGraphIndex(object):
2170
"""A KnitVersionedFiles index layered on GraphIndex."""
2172
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
2174
"""Construct a KnitGraphIndex on a graph_index.
2176
:param graph_index: An implementation of bzrlib.index.GraphIndex.
2177
:param is_locked: A callback to check whether the object should answer
2179
:param deltas: Allow delta-compressed records.
2180
:param parents: If True, record knits parents, if not do not record
2182
:param add_callback: If not None, allow additions to the index and call
2183
this callback with a list of added GraphIndex nodes:
2184
[(node, value, node_refs), ...]
2185
:param is_locked: A callback, returns True if the index is locked and
2188
self._add_callback = add_callback
2189
self._graph_index = graph_index
2190
self._deltas = deltas
2191
self._parents = parents
2192
if deltas and not parents:
2193
# XXX: TODO: Delta tree and parent graph should be conceptually
2195
raise KnitCorrupt(self, "Cannot do delta compression without "
2197
self.has_graph = parents
2198
self._is_locked = is_locked
2199
self._missing_compression_parents = set()
2202
return "%s(%r)" % (self.__class__.__name__, self._graph_index)
2204
def add_records(self, records, random_id=False):
2205
"""Add multiple records to the index.
2207
This function does not insert data into the Immutable GraphIndex
2208
backing the KnitGraphIndex, instead it prepares data for insertion by
2209
the caller and checks that it is safe to insert then calls
2210
self._add_callback with the prepared GraphIndex nodes.
2212
:param records: a list of tuples:
2213
(key, options, access_memo, parents).
2214
:param random_id: If True the ids being added were randomly generated
2215
and no check for existence will be performed.
2217
if not self._add_callback:
2218
raise errors.ReadOnlyError(self)
2219
# we hope there are no repositories with inconsistent parentage
2223
for (key, options, access_memo, parents) in records:
2225
parents = tuple(parents)
2226
index, pos, size = access_memo
2227
if 'no-eol' in options:
2231
value += "%d %d" % (pos, size)
2232
if not self._deltas:
2233
if 'line-delta' in options:
2234
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
2237
if 'line-delta' in options:
2238
node_refs = (parents, (parents[0],))
2240
node_refs = (parents, ())
2242
node_refs = (parents, )
2245
raise KnitCorrupt(self, "attempt to add node with parents "
2246
"in parentless index.")
2248
keys[key] = (value, node_refs)
2251
present_nodes = self._get_entries(keys)
2252
for (index, key, value, node_refs) in present_nodes:
2253
if (value[0] != keys[key][0][0] or
2254
node_refs[:1] != keys[key][1][:1]):
2255
raise KnitCorrupt(self, "inconsistent details in add_records"
2256
": %s %s" % ((value, node_refs), keys[key]))
2260
for key, (value, node_refs) in keys.iteritems():
2261
result.append((key, value, node_refs))
2263
for key, (value, node_refs) in keys.iteritems():
2264
result.append((key, value))
2265
self._add_callback(result)
2267
def _scan_unvalidated_index(self, graph_index):
2268
"""Inform this _KnitGraphIndex that there is an unvalidated index.
2270
This allows this _KnitGraphIndex to keep track of any missing
2271
compression parents we may want to have filled in to make those
2274
:param graph_index: A GraphIndex
2276
self._missing_compression_parents.update(
2277
graph_index.external_references(ref_list_num=1))
2278
self._missing_compression_parents.difference_update(
2279
self.get_parent_map(self._missing_compression_parents))
2281
def get_missing_compression_parents(self):
2282
"""Return the keys of compression parents missing from unvalidated
2285
return frozenset(self._missing_compression_parents)
2287
def _check_read(self):
2288
"""raise if reads are not permitted."""
2289
if not self._is_locked():
2290
raise errors.ObjectNotLocked(self)
2292
def _check_write_ok(self):
2293
"""Assert if writes are not permitted."""
2294
if not self._is_locked():
2295
raise errors.ObjectNotLocked(self)
2297
def _compression_parent(self, an_entry):
2298
# return the key that an_entry is compressed against, or None
2299
# Grab the second parent list (as deltas implies parents currently)
2300
compression_parents = an_entry[3][1]
2301
if not compression_parents:
2303
if len(compression_parents) != 1:
2304
raise AssertionError(
2305
"Too many compression parents: %r" % compression_parents)
2306
return compression_parents[0]
2308
def get_build_details(self, keys):
2309
"""Get the method, index_memo and compression parent for version_ids.
2311
Ghosts are omitted from the result.
2313
:param keys: An iterable of keys.
2314
:return: A dict of key:
2315
(index_memo, compression_parent, parents, record_details).
2317
opaque structure to pass to read_records to extract the raw
2320
Content that this record is built upon, may be None
2322
Logical parents of this node
2324
extra information about the content which needs to be passed to
2325
Factory.parse_record
2329
entries = self._get_entries(keys, False)
2330
for entry in entries:
2332
if not self._parents:
2335
parents = entry[3][0]
2336
if not self._deltas:
2337
compression_parent_key = None
2339
compression_parent_key = self._compression_parent(entry)
2340
noeol = (entry[2][0] == 'N')
2341
if compression_parent_key:
2342
method = 'line-delta'
2345
result[key] = (self._node_to_position(entry),
2346
compression_parent_key, parents,
2350
def _get_entries(self, keys, check_present=False):
2351
"""Get the entries for keys.
2353
:param keys: An iterable of index key tuples.
2358
for node in self._graph_index.iter_entries(keys):
2360
found_keys.add(node[1])
2362
# adapt parentless index to the rest of the code.
2363
for node in self._graph_index.iter_entries(keys):
2364
yield node[0], node[1], node[2], ()
2365
found_keys.add(node[1])
2367
missing_keys = keys.difference(found_keys)
2369
raise RevisionNotPresent(missing_keys.pop(), self)
2371
def get_method(self, key):
2372
"""Return compression method of specified key."""
2373
return self._get_method(self._get_node(key))
2375
def _get_method(self, node):
2376
if not self._deltas:
2378
if self._compression_parent(node):
2383
def _get_node(self, key):
2385
return list(self._get_entries([key]))[0]
2387
raise RevisionNotPresent(key, self)
2389
def get_options(self, key):
2390
"""Return a list representing options.
2394
node = self._get_node(key)
2395
options = [self._get_method(node)]
2396
if node[2][0] == 'N':
2397
options.append('no-eol')
2400
def get_parent_map(self, keys):
2401
"""Get a map of the parents of keys.
2403
:param keys: The keys to look up parents for.
2404
:return: A mapping from keys to parents. Absent keys are absent from
2408
nodes = self._get_entries(keys)
2412
result[node[1]] = node[3][0]
2415
result[node[1]] = None
2418
def get_position(self, key):
2419
"""Return details needed to access the version.
2421
:return: a tuple (index, data position, size) to hand to the access
2422
logic to get the record.
2424
node = self._get_node(key)
2425
return self._node_to_position(node)
2427
has_key = _mod_index._has_key_from_parent_map
2430
"""Get all the keys in the collection.
2432
The keys are not ordered.
2435
return [node[1] for node in self._graph_index.iter_all_entries()]
2437
missing_keys = _mod_index._missing_keys_from_parent_map
2439
def _node_to_position(self, node):
2440
"""Convert an index value to position details."""
2441
bits = node[2][1:].split(' ')
2442
return node[0], int(bits[0]), int(bits[1])
2444
def _sort_keys_by_io(self, keys, positions):
2445
"""Figure out an optimal order to read the records for the given keys.
2447
Sort keys, grouped by index and sorted by position.
2449
:param keys: A list of keys whose records we want to read. This will be
2451
:param positions: A dict, such as the one returned by
2452
_get_components_positions()
2455
def get_index_memo(key):
2456
# index_memo is at offset [1]. It is made up of (GraphIndex,
2457
# position, size). GI is an object, which will be unique for each
2458
# pack file. This causes us to group by pack file, then sort by
2459
# position. Size doesn't matter, but it isn't worth breaking up the
2461
return positions[key][1]
2462
return keys.sort(key=get_index_memo)
2465
class _KnitKeyAccess(object):
2466
"""Access to records in .knit files."""
2468
def __init__(self, transport, mapper):
2469
"""Create a _KnitKeyAccess with transport and mapper.
2471
:param transport: The transport the access object is rooted at.
2472
:param mapper: The mapper used to map keys to .knit files.
2474
self._transport = transport
2475
self._mapper = mapper
2477
def add_raw_records(self, key_sizes, raw_data):
2478
"""Add raw knit bytes to a storage area.
2480
The data is spooled to the container writer in one bytes-record per
2483
:param sizes: An iterable of tuples containing the key and size of each
2485
:param raw_data: A bytestring containing the data.
2486
:return: A list of memos to retrieve the record later. Each memo is an
2487
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2488
length), where the key is the record key.
2490
if type(raw_data) != str:
2491
raise AssertionError(
2492
'data must be plain bytes was %s' % type(raw_data))
2495
# TODO: This can be tuned for writing to sftp and other servers where
2496
# append() is relatively expensive by grouping the writes to each key
2498
for key, size in key_sizes:
2499
path = self._mapper.map(key)
2501
base = self._transport.append_bytes(path + '.knit',
2502
raw_data[offset:offset+size])
2503
except errors.NoSuchFile:
2504
self._transport.mkdir(osutils.dirname(path))
2505
base = self._transport.append_bytes(path + '.knit',
2506
raw_data[offset:offset+size])
2510
result.append((key, base, size))
2513
def get_raw_records(self, memos_for_retrieval):
2514
"""Get the raw bytes for a records.
2516
:param memos_for_retrieval: An iterable containing the access memo for
2517
retrieving the bytes.
2518
:return: An iterator over the bytes of the records.
2520
# first pass, group into same-index request to minimise readv's issued.
2522
current_prefix = None
2523
for (key, offset, length) in memos_for_retrieval:
2524
if current_prefix == key[:-1]:
2525
current_list.append((offset, length))
2527
if current_prefix is not None:
2528
request_lists.append((current_prefix, current_list))
2529
current_prefix = key[:-1]
2530
current_list = [(offset, length)]
2531
# handle the last entry
2532
if current_prefix is not None:
2533
request_lists.append((current_prefix, current_list))
2534
for prefix, read_vector in request_lists:
2535
path = self._mapper.map(prefix) + '.knit'
2536
for pos, data in self._transport.readv(path, read_vector):
2540
class _DirectPackAccess(object):
2541
"""Access to data in one or more packs with less translation."""
2543
def __init__(self, index_to_packs, reload_func=None):
2544
"""Create a _DirectPackAccess object.
2546
:param index_to_packs: A dict mapping index objects to the transport
2547
and file names for obtaining data.
2548
:param reload_func: A function to call if we determine that the pack
2549
files have moved and we need to reload our caches. See
2550
bzrlib.repo_fmt.pack_repo.AggregateIndex for more details.
2552
self._container_writer = None
2553
self._write_index = None
2554
self._indices = index_to_packs
2555
self._reload_func = reload_func
2557
def add_raw_records(self, key_sizes, raw_data):
2558
"""Add raw knit bytes to a storage area.
2560
The data is spooled to the container writer in one bytes-record per
2563
:param sizes: An iterable of tuples containing the key and size of each
2565
:param raw_data: A bytestring containing the data.
2566
:return: A list of memos to retrieve the record later. Each memo is an
2567
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2568
length), where the index field is the write_index object supplied
2569
to the PackAccess object.
2571
if type(raw_data) != str:
2572
raise AssertionError(
2573
'data must be plain bytes was %s' % type(raw_data))
2576
for key, size in key_sizes:
2577
p_offset, p_length = self._container_writer.add_bytes_record(
2578
raw_data[offset:offset+size], [])
2580
result.append((self._write_index, p_offset, p_length))
2583
def get_raw_records(self, memos_for_retrieval):
2584
"""Get the raw bytes for a records.
2586
:param memos_for_retrieval: An iterable containing the (index, pos,
2587
length) memo for retrieving the bytes. The Pack access method
2588
looks up the pack to use for a given record in its index_to_pack
2590
:return: An iterator over the bytes of the records.
2592
# first pass, group into same-index requests
2594
current_index = None
2595
for (index, offset, length) in memos_for_retrieval:
2596
if current_index == index:
2597
current_list.append((offset, length))
2599
if current_index is not None:
2600
request_lists.append((current_index, current_list))
2601
current_index = index
2602
current_list = [(offset, length)]
2603
# handle the last entry
2604
if current_index is not None:
2605
request_lists.append((current_index, current_list))
2606
for index, offsets in request_lists:
2608
transport, path = self._indices[index]
2610
# A KeyError here indicates that someone has triggered an index
2611
# reload, and this index has gone missing, we need to start
2613
if self._reload_func is None:
2614
# If we don't have a _reload_func there is nothing that can
2617
raise errors.RetryWithNewPacks(index,
2618
reload_occurred=True,
2619
exc_info=sys.exc_info())
2621
reader = pack.make_readv_reader(transport, path, offsets)
2622
for names, read_func in reader.iter_records():
2623
yield read_func(None)
2624
except errors.NoSuchFile:
2625
# A NoSuchFile error indicates that a pack file has gone
2626
# missing on disk, we need to trigger a reload, and start over.
2627
if self._reload_func is None:
2629
raise errors.RetryWithNewPacks(transport.abspath(path),
2630
reload_occurred=False,
2631
exc_info=sys.exc_info())
2633
def set_writer(self, writer, index, transport_packname):
2634
"""Set a writer to use for adding data."""
2635
if index is not None:
2636
self._indices[index] = transport_packname
2637
self._container_writer = writer
2638
self._write_index = index
2640
def reload_or_raise(self, retry_exc):
2641
"""Try calling the reload function, or re-raise the original exception.
2643
This should be called after _DirectPackAccess raises a
2644
RetryWithNewPacks exception. This function will handle the common logic
2645
of determining when the error is fatal versus being temporary.
2646
It will also make sure that the original exception is raised, rather
2647
than the RetryWithNewPacks exception.
2649
If this function returns, then the calling function should retry
2650
whatever operation was being performed. Otherwise an exception will
2653
:param retry_exc: A RetryWithNewPacks exception.
2656
if self._reload_func is None:
2658
elif not self._reload_func():
2659
# The reload claimed that nothing changed
2660
if not retry_exc.reload_occurred:
2661
# If there wasn't an earlier reload, then we really were
2662
# expecting to find changes. We didn't find them, so this is a
2666
exc_class, exc_value, exc_traceback = retry_exc.exc_info
2667
raise exc_class, exc_value, exc_traceback
2670
# Deprecated, use PatienceSequenceMatcher instead
2671
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2674
def annotate_knit(knit, revision_id):
2675
"""Annotate a knit with no cached annotations.
2677
This implementation is for knits with no cached annotations.
2678
It will work for knits with cached annotations, but this is not
2681
annotator = _KnitAnnotator(knit)
2682
return iter(annotator.annotate(revision_id))
2685
class _KnitAnnotator(object):
2686
"""Build up the annotations for a text."""
2688
def __init__(self, knit):
2691
# Content objects, differs from fulltexts because of how final newlines
2692
# are treated by knits. the content objects here will always have a
2694
self._fulltext_contents = {}
2696
# Annotated lines of specific revisions
2697
self._annotated_lines = {}
2699
# Track the raw data for nodes that we could not process yet.
2700
# This maps the revision_id of the base to a list of children that will
2701
# annotated from it.
2702
self._pending_children = {}
2704
# Nodes which cannot be extracted
2705
self._ghosts = set()
2707
# Track how many children this node has, so we know if we need to keep
2709
self._annotate_children = {}
2710
self._compression_children = {}
2712
self._all_build_details = {}
2713
# The children => parent revision_id graph
2714
self._revision_id_graph = {}
2716
self._heads_provider = None
2718
self._nodes_to_keep_annotations = set()
2719
self._generations_until_keep = 100
2721
def set_generations_until_keep(self, value):
2722
"""Set the number of generations before caching a node.
2724
Setting this to -1 will cache every merge node, setting this higher
2725
will cache fewer nodes.
2727
self._generations_until_keep = value
2729
def _add_fulltext_content(self, revision_id, content_obj):
2730
self._fulltext_contents[revision_id] = content_obj
2731
# TODO: jam 20080305 It might be good to check the sha1digest here
2732
return content_obj.text()
2734
def _check_parents(self, child, nodes_to_annotate):
2735
"""Check if all parents have been processed.
2737
:param child: A tuple of (rev_id, parents, raw_content)
2738
:param nodes_to_annotate: If child is ready, add it to
2739
nodes_to_annotate, otherwise put it back in self._pending_children
2741
for parent_id in child[1]:
2742
if (parent_id not in self._annotated_lines):
2743
# This parent is present, but another parent is missing
2744
self._pending_children.setdefault(parent_id,
2748
# This one is ready to be processed
2749
nodes_to_annotate.append(child)
2751
def _add_annotation(self, revision_id, fulltext, parent_ids,
2752
left_matching_blocks=None):
2753
"""Add an annotation entry.
2755
All parents should already have been annotated.
2756
:return: A list of children that now have their parents satisfied.
2758
a = self._annotated_lines
2759
annotated_parent_lines = [a[p] for p in parent_ids]
2760
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2761
fulltext, revision_id, left_matching_blocks,
2762
heads_provider=self._get_heads_provider()))
2763
self._annotated_lines[revision_id] = annotated_lines
2764
for p in parent_ids:
2765
ann_children = self._annotate_children[p]
2766
ann_children.remove(revision_id)
2767
if (not ann_children
2768
and p not in self._nodes_to_keep_annotations):
2769
del self._annotated_lines[p]
2770
del self._all_build_details[p]
2771
if p in self._fulltext_contents:
2772
del self._fulltext_contents[p]
2773
# Now that we've added this one, see if there are any pending
2774
# deltas to be done, certainly this parent is finished
2775
nodes_to_annotate = []
2776
for child in self._pending_children.pop(revision_id, []):
2777
self._check_parents(child, nodes_to_annotate)
2778
return nodes_to_annotate
2780
def _get_build_graph(self, key):
2781
"""Get the graphs for building texts and annotations.
2783
The data you need for creating a full text may be different than the
2784
data you need to annotate that text. (At a minimum, you need both
2785
parents to create an annotation, but only need 1 parent to generate the
2788
:return: A list of (key, index_memo) records, suitable for
2789
passing to read_records_iter to start reading in the raw data fro/
2792
if key in self._annotated_lines:
2795
pending = set([key])
2800
# get all pending nodes
2802
this_iteration = pending
2803
build_details = self._knit._index.get_build_details(this_iteration)
2804
self._all_build_details.update(build_details)
2805
# new_nodes = self._knit._index._get_entries(this_iteration)
2807
for key, details in build_details.iteritems():
2808
(index_memo, compression_parent, parents,
2809
record_details) = details
2810
self._revision_id_graph[key] = parents
2811
records.append((key, index_memo))
2812
# Do we actually need to check _annotated_lines?
2813
pending.update(p for p in parents
2814
if p not in self._all_build_details)
2815
if compression_parent:
2816
self._compression_children.setdefault(compression_parent,
2819
for parent in parents:
2820
self._annotate_children.setdefault(parent,
2822
num_gens = generation - kept_generation
2823
if ((num_gens >= self._generations_until_keep)
2824
and len(parents) > 1):
2825
kept_generation = generation
2826
self._nodes_to_keep_annotations.add(key)
2828
missing_versions = this_iteration.difference(build_details.keys())
2829
self._ghosts.update(missing_versions)
2830
for missing_version in missing_versions:
2831
# add a key, no parents
2832
self._revision_id_graph[missing_version] = ()
2833
pending.discard(missing_version) # don't look for it
2834
if self._ghosts.intersection(self._compression_children):
2836
"We cannot have nodes which have a ghost compression parent:\n"
2838
"compression children: %r"
2839
% (self._ghosts, self._compression_children))
2840
# Cleanout anything that depends on a ghost so that we don't wait for
2841
# the ghost to show up
2842
for node in self._ghosts:
2843
if node in self._annotate_children:
2844
# We won't be building this node
2845
del self._annotate_children[node]
2846
# Generally we will want to read the records in reverse order, because
2847
# we find the parent nodes after the children
2851
def _annotate_records(self, records):
2852
"""Build the annotations for the listed records."""
2853
# We iterate in the order read, rather than a strict order requested
2854
# However, process what we can, and put off to the side things that
2855
# still need parents, cleaning them up when those parents are
2857
for (rev_id, record,
2858
digest) in self._knit._read_records_iter(records):
2859
if rev_id in self._annotated_lines:
2861
parent_ids = self._revision_id_graph[rev_id]
2862
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2863
details = self._all_build_details[rev_id]
2864
(index_memo, compression_parent, parents,
2865
record_details) = details
2866
nodes_to_annotate = []
2867
# TODO: Remove the punning between compression parents, and
2868
# parent_ids, we should be able to do this without assuming
2870
if len(parent_ids) == 0:
2871
# There are no parents for this node, so just add it
2872
# TODO: This probably needs to be decoupled
2873
fulltext_content, delta = self._knit._factory.parse_record(
2874
rev_id, record, record_details, None)
2875
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2876
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2877
parent_ids, left_matching_blocks=None))
2879
child = (rev_id, parent_ids, record)
2880
# Check if all the parents are present
2881
self._check_parents(child, nodes_to_annotate)
2882
while nodes_to_annotate:
2883
# Should we use a queue here instead of a stack?
2884
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2885
(index_memo, compression_parent, parents,
2886
record_details) = self._all_build_details[rev_id]
2888
if compression_parent is not None:
2889
comp_children = self._compression_children[compression_parent]
2890
if rev_id not in comp_children:
2891
raise AssertionError("%r not in compression children %r"
2892
% (rev_id, comp_children))
2893
# If there is only 1 child, it is safe to reuse this
2895
reuse_content = (len(comp_children) == 1
2896
and compression_parent not in
2897
self._nodes_to_keep_annotations)
2899
# Remove it from the cache since it will be changing
2900
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2901
# Make sure to copy the fulltext since it might be
2903
parent_fulltext = list(parent_fulltext_content.text())
2905
parent_fulltext_content = self._fulltext_contents[compression_parent]
2906
parent_fulltext = parent_fulltext_content.text()
2907
comp_children.remove(rev_id)
2908
fulltext_content, delta = self._knit._factory.parse_record(
2909
rev_id, record, record_details,
2910
parent_fulltext_content,
2911
copy_base_content=(not reuse_content))
2912
fulltext = self._add_fulltext_content(rev_id,
2914
if compression_parent == parent_ids[0]:
2915
# the compression_parent is the left parent, so we can
2917
blocks = KnitContent.get_line_delta_blocks(delta,
2918
parent_fulltext, fulltext)
2920
fulltext_content = self._knit._factory.parse_fulltext(
2922
fulltext = self._add_fulltext_content(rev_id,
2924
nodes_to_annotate.extend(
2925
self._add_annotation(rev_id, fulltext, parent_ids,
2926
left_matching_blocks=blocks))
2928
def _get_heads_provider(self):
2929
"""Create a heads provider for resolving ancestry issues."""
2930
if self._heads_provider is not None:
2931
return self._heads_provider
2932
parent_provider = _mod_graph.DictParentsProvider(
2933
self._revision_id_graph)
2934
graph_obj = _mod_graph.Graph(parent_provider)
2935
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2936
self._heads_provider = head_cache
2939
def annotate(self, key):
2940
"""Return the annotated fulltext at the given key.
2942
:param key: The key to annotate.
2944
if len(self._knit._fallback_vfs) > 0:
2945
# stacked knits can't use the fast path at present.
2946
return self._simple_annotate(key)
2949
records = self._get_build_graph(key)
2950
if key in self._ghosts:
2951
raise errors.RevisionNotPresent(key, self._knit)
2952
self._annotate_records(records)
2953
return self._annotated_lines[key]
2954
except errors.RetryWithNewPacks, e:
2955
self._knit._access.reload_or_raise(e)
2956
# The cached build_details are no longer valid
2957
self._all_build_details.clear()
2959
def _simple_annotate(self, key):
2960
"""Return annotated fulltext, rediffing from the full texts.
2962
This is slow but makes no assumptions about the repository
2963
being able to produce line deltas.
2965
# TODO: this code generates a parent maps of present ancestors; it
2966
# could be split out into a separate method, and probably should use
2967
# iter_ancestry instead. -- mbp and robertc 20080704
2968
graph = _mod_graph.Graph(self._knit)
2969
head_cache = _mod_graph.FrozenHeadsCache(graph)
2970
search = graph._make_breadth_first_searcher([key])
2974
present, ghosts = search.next_with_ghosts()
2975
except StopIteration:
2977
keys.update(present)
2978
parent_map = self._knit.get_parent_map(keys)
2980
reannotate = annotate.reannotate
2981
for record in self._knit.get_record_stream(keys, 'topological', True):
2983
fulltext = osutils.chunks_to_lines(record.get_bytes_as('chunked'))
2984
parents = parent_map[key]
2985
if parents is not None:
2986
parent_lines = [parent_cache[parent] for parent in parent_map[key]]
2989
parent_cache[key] = list(
2990
reannotate(parent_lines, fulltext, key, None, head_cache))
2992
return parent_cache[key]
2994
raise errors.RevisionNotPresent(key, self._knit)
2998
from bzrlib._knit_load_data_c import _load_data_c as _load_data
3000
from bzrlib._knit_load_data_py import _load_data_py as _load_data
added
removed
bzrlib/knit.py
