1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Knit versionedfile implementation.
19
A knit is a versioned file implementation that supports efficient append only
23
lifeless: the data file is made up of "delta records". each delta record has a delta header
24
that contains; (1) a version id, (2) the size of the delta (in lines), and (3) the digest of
25
the -expanded data- (ie, the delta applied to the parent). the delta also ends with a
26
end-marker; simply "end VERSION"
28
delta can be line or full contents.a
29
... the 8's there are the index number of the annotation.
30
version robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad 7 c7d23b2a5bd6ca00e8e266cec0ec228158ee9f9e
34
8 e.set('executable', 'yes')
36
8 if elt.get('executable') == 'yes':
37
8 ie.executable = True
38
end robertc@robertcollins.net-20051003014215-ee2990904cc4c7ad
42
09:33 < jrydberg> lifeless: each index is made up of a tuple of; version id, options, position, size, parents
43
09:33 < jrydberg> lifeless: the parents are currently dictionary compressed
44
09:33 < jrydberg> lifeless: (meaning it currently does not support ghosts)
45
09:33 < lifeless> right
46
09:33 < jrydberg> lifeless: the position and size is the range in the data file
49
so the index sequence is the dictionary compressed sequence number used
50
in the deltas to provide line annotation
55
# 10:16 < lifeless> make partial index writes safe
56
# 10:16 < lifeless> implement 'knit.check()' like weave.check()
57
# 10:17 < lifeless> record known ghosts so we can detect when they are filled in rather than the current 'reweave
59
# move sha1 out of the content so that join is faster at verifying parents
60
# record content length ?
64
from cStringIO import StringIO
65
from itertools import izip, chain
71
from zlib import Z_DEFAULT_COMPRESSION
74
from bzrlib.lazy_import import lazy_import
75
lazy_import(globals(), """
96
from bzrlib.errors import (
104
RevisionAlreadyPresent,
106
from bzrlib.graph import Graph
107
from bzrlib.osutils import (
114
from bzrlib.tsort import topo_sort
115
from bzrlib.tuned_gzip import GzipFile, bytes_to_gzip
117
from bzrlib.versionedfile import (
118
AbsentContentFactory,
122
FulltextContentFactory,
129
# TODO: Split out code specific to this format into an associated object.
131
# TODO: Can we put in some kind of value to check that the index and data
132
# files belong together?
134
# TODO: accommodate binaries, perhaps by storing a byte count
136
# TODO: function to check whole file
138
# TODO: atomically append data, then measure backwards from the cursor
139
# position after writing to work out where it was located. we may need to
140
# bypass python file buffering.
142
DATA_SUFFIX = '.knit'
143
INDEX_SUFFIX = '.kndx'
146
class KnitAdapter(object):
147
"""Base class for knit record adaption."""
149
def __init__(self, basis_vf):
150
"""Create an adapter which accesses full texts from basis_vf.
152
:param basis_vf: A versioned file to access basis texts of deltas from.
153
May be None for adapters that do not need to access basis texts.
155
self._data = KnitVersionedFiles(None, None)
156
self._annotate_factory = KnitAnnotateFactory()
157
self._plain_factory = KnitPlainFactory()
158
self._basis_vf = basis_vf
161
class FTAnnotatedToUnannotated(KnitAdapter):
162
"""An adapter from FT annotated knits to unannotated ones."""
164
def get_bytes(self, factory, annotated_compressed_bytes):
166
self._data._parse_record_unchecked(annotated_compressed_bytes)
167
content = self._annotate_factory.parse_fulltext(contents, rec[1])
168
size, bytes = self._data._record_to_data((rec[1],), rec[3], content.text())
172
class DeltaAnnotatedToUnannotated(KnitAdapter):
173
"""An adapter for deltas from annotated to unannotated."""
175
def get_bytes(self, factory, annotated_compressed_bytes):
177
self._data._parse_record_unchecked(annotated_compressed_bytes)
178
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
180
contents = self._plain_factory.lower_line_delta(delta)
181
size, bytes = self._data._record_to_data((rec[1],), rec[3], contents)
185
class FTAnnotatedToFullText(KnitAdapter):
186
"""An adapter from FT annotated knits to unannotated ones."""
188
def get_bytes(self, factory, annotated_compressed_bytes):
190
self._data._parse_record_unchecked(annotated_compressed_bytes)
191
content, delta = self._annotate_factory.parse_record(factory.key[-1],
192
contents, factory._build_details, None)
193
return ''.join(content.text())
196
class DeltaAnnotatedToFullText(KnitAdapter):
197
"""An adapter for deltas from annotated to unannotated."""
199
def get_bytes(self, factory, annotated_compressed_bytes):
201
self._data._parse_record_unchecked(annotated_compressed_bytes)
202
delta = self._annotate_factory.parse_line_delta(contents, rec[1],
204
compression_parent = factory.parents[0]
205
basis_entry = self._basis_vf.get_record_stream(
206
[compression_parent], 'unordered', True).next()
207
if basis_entry.storage_kind == 'absent':
208
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
209
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
210
# Manually apply the delta because we have one annotated content and
212
basis_content = PlainKnitContent(basis_lines, compression_parent)
213
basis_content.apply_delta(delta, rec[1])
214
basis_content._should_strip_eol = factory._build_details[1]
215
return ''.join(basis_content.text())
218
class FTPlainToFullText(KnitAdapter):
219
"""An adapter from FT plain knits to unannotated ones."""
221
def get_bytes(self, factory, compressed_bytes):
223
self._data._parse_record_unchecked(compressed_bytes)
224
content, delta = self._plain_factory.parse_record(factory.key[-1],
225
contents, factory._build_details, None)
226
return ''.join(content.text())
229
class DeltaPlainToFullText(KnitAdapter):
230
"""An adapter for deltas from annotated to unannotated."""
232
def get_bytes(self, factory, compressed_bytes):
234
self._data._parse_record_unchecked(compressed_bytes)
235
delta = self._plain_factory.parse_line_delta(contents, rec[1])
236
compression_parent = factory.parents[0]
237
# XXX: string splitting overhead.
238
basis_entry = self._basis_vf.get_record_stream(
239
[compression_parent], 'unordered', True).next()
240
if basis_entry.storage_kind == 'absent':
241
raise errors.RevisionNotPresent(compression_parent, self._basis_vf)
242
basis_lines = split_lines(basis_entry.get_bytes_as('fulltext'))
243
basis_content = PlainKnitContent(basis_lines, compression_parent)
244
# Manually apply the delta because we have one annotated content and
246
content, _ = self._plain_factory.parse_record(rec[1], contents,
247
factory._build_details, basis_content)
248
return ''.join(content.text())
251
class KnitContentFactory(ContentFactory):
252
"""Content factory for streaming from knits.
254
:seealso ContentFactory:
257
def __init__(self, key, parents, build_details, sha1, raw_record,
258
annotated, knit=None):
259
"""Create a KnitContentFactory for key.
262
:param parents: The parents.
263
:param build_details: The build details as returned from
265
:param sha1: The sha1 expected from the full text of this object.
266
:param raw_record: The bytes of the knit data from disk.
267
:param annotated: True if the raw data is annotated.
269
ContentFactory.__init__(self)
272
self.parents = parents
273
if build_details[0] == 'line-delta':
278
annotated_kind = 'annotated-'
281
self.storage_kind = 'knit-%s%s-gz' % (annotated_kind, kind)
282
self._raw_record = raw_record
283
self._build_details = build_details
286
def get_bytes_as(self, storage_kind):
287
if storage_kind == self.storage_kind:
288
return self._raw_record
289
if storage_kind == 'fulltext' and self._knit is not None:
290
return self._knit.get_text(self.key[0])
292
raise errors.UnavailableRepresentation(self.key, storage_kind,
296
class KnitContent(object):
297
"""Content of a knit version to which deltas can be applied."""
300
self._should_strip_eol = False
302
def apply_delta(self, delta, new_version_id):
303
"""Apply delta to this object to become new_version_id."""
304
raise NotImplementedError(self.apply_delta)
306
def cleanup_eol(self, copy_on_mutate=True):
307
if self._should_strip_eol:
309
self._lines = self._lines[:]
310
self.strip_last_line_newline()
312
def line_delta_iter(self, new_lines):
313
"""Generate line-based delta from this content to new_lines."""
314
new_texts = new_lines.text()
315
old_texts = self.text()
316
s = patiencediff.PatienceSequenceMatcher(None, old_texts, new_texts)
317
for tag, i1, i2, j1, j2 in s.get_opcodes():
320
# ofrom, oto, length, data
321
yield i1, i2, j2 - j1, new_lines._lines[j1:j2]
323
def line_delta(self, new_lines):
324
return list(self.line_delta_iter(new_lines))
327
def get_line_delta_blocks(knit_delta, source, target):
328
"""Extract SequenceMatcher.get_matching_blocks() from a knit delta"""
329
target_len = len(target)
332
for s_begin, s_end, t_len, new_text in knit_delta:
333
true_n = s_begin - s_pos
336
# knit deltas do not provide reliable info about whether the
337
# last line of a file matches, due to eol handling.
338
if source[s_pos + n -1] != target[t_pos + n -1]:
341
yield s_pos, t_pos, n
342
t_pos += t_len + true_n
344
n = target_len - t_pos
346
if source[s_pos + n -1] != target[t_pos + n -1]:
349
yield s_pos, t_pos, n
350
yield s_pos + (target_len - t_pos), target_len, 0
353
class AnnotatedKnitContent(KnitContent):
354
"""Annotated content."""
356
def __init__(self, lines):
357
KnitContent.__init__(self)
361
"""Return a list of (origin, text) for each content line."""
362
return list(self._lines)
364
def apply_delta(self, delta, new_version_id):
365
"""Apply delta to this object to become new_version_id."""
368
for start, end, count, delta_lines in delta:
369
lines[offset+start:offset+end] = delta_lines
370
offset = offset + (start - end) + count
372
def strip_last_line_newline(self):
373
line = self._lines[-1][1].rstrip('\n')
374
self._lines[-1] = (self._lines[-1][0], line)
375
self._should_strip_eol = False
379
lines = [text for origin, text in self._lines]
380
except ValueError, e:
381
# most commonly (only?) caused by the internal form of the knit
382
# missing annotation information because of a bug - see thread
384
raise KnitCorrupt(self,
385
"line in annotated knit missing annotation information: %s"
388
if self._should_strip_eol:
389
lines[-1] = lines[-1].rstrip('\n')
393
return AnnotatedKnitContent(self._lines[:])
396
class PlainKnitContent(KnitContent):
397
"""Unannotated content.
399
When annotate[_iter] is called on this content, the same version is reported
400
for all lines. Generally, annotate[_iter] is not useful on PlainKnitContent
404
def __init__(self, lines, version_id):
405
KnitContent.__init__(self)
407
self._version_id = version_id
410
"""Return a list of (origin, text) for each content line."""
411
return [(self._version_id, line) for line in self._lines]
413
def apply_delta(self, delta, new_version_id):
414
"""Apply delta to this object to become new_version_id."""
417
for start, end, count, delta_lines in delta:
418
lines[offset+start:offset+end] = delta_lines
419
offset = offset + (start - end) + count
420
self._version_id = new_version_id
423
return PlainKnitContent(self._lines[:], self._version_id)
425
def strip_last_line_newline(self):
426
self._lines[-1] = self._lines[-1].rstrip('\n')
427
self._should_strip_eol = False
431
if self._should_strip_eol:
433
lines[-1] = lines[-1].rstrip('\n')
437
class _KnitFactory(object):
438
"""Base class for common Factory functions."""
440
def parse_record(self, version_id, record, record_details,
441
base_content, copy_base_content=True):
442
"""Parse a record into a full content object.
444
:param version_id: The official version id for this content
445
:param record: The data returned by read_records_iter()
446
:param record_details: Details about the record returned by
448
:param base_content: If get_build_details returns a compression_parent,
449
you must return a base_content here, else use None
450
:param copy_base_content: When building from the base_content, decide
451
you can either copy it and return a new object, or modify it in
453
:return: (content, delta) A Content object and possibly a line-delta,
456
method, noeol = record_details
457
if method == 'line-delta':
458
if copy_base_content:
459
content = base_content.copy()
461
content = base_content
462
delta = self.parse_line_delta(record, version_id)
463
content.apply_delta(delta, version_id)
465
content = self.parse_fulltext(record, version_id)
467
content._should_strip_eol = noeol
468
return (content, delta)
471
class KnitAnnotateFactory(_KnitFactory):
472
"""Factory for creating annotated Content objects."""
476
def make(self, lines, version_id):
477
num_lines = len(lines)
478
return AnnotatedKnitContent(zip([version_id] * num_lines, lines))
480
def parse_fulltext(self, content, version_id):
481
"""Convert fulltext to internal representation
483
fulltext content is of the format
484
revid(utf8) plaintext\n
485
internal representation is of the format:
488
# TODO: jam 20070209 The tests expect this to be returned as tuples,
489
# but the code itself doesn't really depend on that.
490
# Figure out a way to not require the overhead of turning the
491
# list back into tuples.
492
lines = [tuple(line.split(' ', 1)) for line in content]
493
return AnnotatedKnitContent(lines)
495
def parse_line_delta_iter(self, lines):
496
return iter(self.parse_line_delta(lines))
498
def parse_line_delta(self, lines, version_id, plain=False):
499
"""Convert a line based delta into internal representation.
501
line delta is in the form of:
502
intstart intend intcount
504
revid(utf8) newline\n
505
internal representation is
506
(start, end, count, [1..count tuples (revid, newline)])
508
:param plain: If True, the lines are returned as a plain
509
list without annotations, not as a list of (origin, content) tuples, i.e.
510
(start, end, count, [1..count newline])
517
def cache_and_return(line):
518
origin, text = line.split(' ', 1)
519
return cache.setdefault(origin, origin), text
521
# walk through the lines parsing.
522
# Note that the plain test is explicitly pulled out of the
523
# loop to minimise any performance impact
526
start, end, count = [int(n) for n in header.split(',')]
527
contents = [next().split(' ', 1)[1] for i in xrange(count)]
528
result.append((start, end, count, contents))
531
start, end, count = [int(n) for n in header.split(',')]
532
contents = [tuple(next().split(' ', 1)) for i in xrange(count)]
533
result.append((start, end, count, contents))
536
def get_fulltext_content(self, lines):
537
"""Extract just the content lines from a fulltext."""
538
return (line.split(' ', 1)[1] for line in lines)
540
def get_linedelta_content(self, lines):
541
"""Extract just the content from a line delta.
543
This doesn't return all of the extra information stored in a delta.
544
Only the actual content lines.
549
header = header.split(',')
550
count = int(header[2])
551
for i in xrange(count):
552
origin, text = next().split(' ', 1)
555
def lower_fulltext(self, content):
556
"""convert a fulltext content record into a serializable form.
558
see parse_fulltext which this inverts.
560
# TODO: jam 20070209 We only do the caching thing to make sure that
561
# the origin is a valid utf-8 line, eventually we could remove it
562
return ['%s %s' % (o, t) for o, t in content._lines]
564
def lower_line_delta(self, delta):
565
"""convert a delta into a serializable form.
567
See parse_line_delta which this inverts.
569
# TODO: jam 20070209 We only do the caching thing to make sure that
570
# the origin is a valid utf-8 line, eventually we could remove it
572
for start, end, c, lines in delta:
573
out.append('%d,%d,%d\n' % (start, end, c))
574
out.extend(origin + ' ' + text
575
for origin, text in lines)
578
def annotate(self, knit, key):
579
content = knit._get_content(key)
580
# adjust for the fact that serialised annotations are only key suffixes
582
if type(key) == tuple:
584
origins = content.annotate()
586
for origin, line in origins:
587
result.append((prefix + (origin,), line))
590
return content.annotate()
593
class KnitPlainFactory(_KnitFactory):
594
"""Factory for creating plain Content objects."""
598
def make(self, lines, version_id):
599
return PlainKnitContent(lines, version_id)
601
def parse_fulltext(self, content, version_id):
602
"""This parses an unannotated fulltext.
604
Note that this is not a noop - the internal representation
605
has (versionid, line) - its just a constant versionid.
607
return self.make(content, version_id)
609
def parse_line_delta_iter(self, lines, version_id):
611
num_lines = len(lines)
612
while cur < num_lines:
615
start, end, c = [int(n) for n in header.split(',')]
616
yield start, end, c, lines[cur:cur+c]
619
def parse_line_delta(self, lines, version_id):
620
return list(self.parse_line_delta_iter(lines, version_id))
622
def get_fulltext_content(self, lines):
623
"""Extract just the content lines from a fulltext."""
626
def get_linedelta_content(self, lines):
627
"""Extract just the content from a line delta.
629
This doesn't return all of the extra information stored in a delta.
630
Only the actual content lines.
635
header = header.split(',')
636
count = int(header[2])
637
for i in xrange(count):
640
def lower_fulltext(self, content):
641
return content.text()
643
def lower_line_delta(self, delta):
645
for start, end, c, lines in delta:
646
out.append('%d,%d,%d\n' % (start, end, c))
650
def annotate(self, knit, key):
651
annotator = _KnitAnnotator(knit)
652
return annotator.annotate(key)
656
def make_file_factory(annotated, mapper):
657
"""Create a factory for creating a file based KnitVersionedFiles.
659
:param annotated: knit annotations are wanted.
660
:param mapper: The mapper from keys to paths.
662
def factory(transport):
663
index = _KndxIndex(transport, mapper, lambda:None, lambda:True, lambda:True)
664
access = _KnitKeyAccess(transport, mapper)
665
return KnitVersionedFiles(index, access, annotated=annotated)
669
def make_pack_factory(graph, delta, keylength):
670
"""Create a factory for creating a pack based VersionedFiles.
672
This is only functional enough to run interface tests, it doesn't try to
673
provide a full pack environment.
675
:param graph: Store a graph.
676
:param delta: Delta compress contents.
677
:param keylength: How long should keys be.
679
def factory(transport):
680
parents = graph or delta
686
max_delta_chain = 200
689
graph_index = _mod_index.InMemoryGraphIndex(reference_lists=ref_length,
690
key_elements=keylength)
691
stream = transport.open_write_stream('newpack')
692
writer = pack.ContainerWriter(stream.write)
694
index = _KnitGraphIndex(graph_index, lambda:True, parents=parents,
695
deltas=delta, add_callback=graph_index.add_nodes)
696
access = _DirectPackAccess({})
697
access.set_writer(writer, graph_index, (transport, 'newpack'))
698
result = KnitVersionedFiles(index, access,
699
max_delta_chain=max_delta_chain)
700
result.stream = stream
701
result.writer = writer
706
def cleanup_pack_knit(versioned_files):
707
versioned_files.stream.close()
708
versioned_files.writer.end()
711
class KnitVersionedFiles(VersionedFiles):
712
"""Storage for many versioned files using knit compression.
714
Backend storage is managed by indices and data objects.
717
def __init__(self, index, data_access, max_delta_chain=200,
719
"""Create a KnitVersionedFiles with index and data_access.
721
:param index: The index for the knit data.
722
:param data_access: The access object to store and retrieve knit
724
:param max_delta_chain: The maximum number of deltas to permit during
725
insertion. Set to 0 to prohibit the use of deltas.
726
:param annotated: Set to True to cause annotations to be calculated and
727
stored during insertion.
730
self._access = data_access
731
self._max_delta_chain = max_delta_chain
733
self._factory = KnitAnnotateFactory()
735
self._factory = KnitPlainFactory()
737
def add_lines(self, key, parents, lines, parent_texts=None,
738
left_matching_blocks=None, nostore_sha=None, random_id=False,
740
"""See VersionedFiles.add_lines()."""
741
self._index._check_write_ok()
742
self._check_add(key, lines, random_id, check_content)
744
# For no-graph knits, have the public interface use None for
747
return self._add(key, lines, parents,
748
parent_texts, left_matching_blocks, nostore_sha, random_id)
750
def _add(self, key, lines, parents, parent_texts,
751
left_matching_blocks, nostore_sha, random_id):
752
"""Add a set of lines on top of version specified by parents.
754
Any versions not present will be converted into ghosts.
756
# first thing, if the content is something we don't need to store, find
758
line_bytes = ''.join(lines)
759
digest = sha_string(line_bytes)
760
if nostore_sha == digest:
761
raise errors.ExistingContent
764
if parent_texts is None:
766
# Do a single query to ascertain parent presence.
767
present_parent_map = self.get_parent_map(parents)
768
for parent in parents:
769
if parent in present_parent_map:
770
present_parents.append(parent)
772
# Currently we can only compress against the left most present parent.
773
if (len(present_parents) == 0 or
774
present_parents[0] != parents[0]):
777
# To speed the extract of texts the delta chain is limited
778
# to a fixed number of deltas. This should minimize both
779
# I/O and the time spend applying deltas.
780
delta = self._check_should_delta(present_parents[0])
782
text_length = len(line_bytes)
785
if lines[-1][-1] != '\n':
786
# copy the contents of lines.
788
options.append('no-eol')
789
lines[-1] = lines[-1] + '\n'
793
if type(element) != str:
794
raise TypeError("key contains non-strings: %r" % (key,))
795
# Knit hunks are still last-element only
797
content = self._factory.make(lines, version_id)
798
if 'no-eol' in options:
799
# Hint to the content object that its text() call should strip the
801
content._should_strip_eol = True
802
if delta or (self._factory.annotated and len(present_parents) > 0):
803
# Merge annotations from parent texts if needed.
804
delta_hunks = self._merge_annotations(content, present_parents,
805
parent_texts, delta, self._factory.annotated,
806
left_matching_blocks)
809
options.append('line-delta')
810
store_lines = self._factory.lower_line_delta(delta_hunks)
811
size, bytes = self._record_to_data(key, digest,
814
options.append('fulltext')
815
# isinstance is slower and we have no hierarchy.
816
if self._factory.__class__ == KnitPlainFactory:
817
# Use the already joined bytes saving iteration time in
819
size, bytes = self._record_to_data(key, digest,
822
# get mixed annotation + content and feed it into the
824
store_lines = self._factory.lower_fulltext(content)
825
size, bytes = self._record_to_data(key, digest,
828
access_memo = self._access.add_raw_records([(key, size)], bytes)[0]
829
self._index.add_records(
830
((key, options, access_memo, parents),),
832
return digest, text_length, content
834
def annotate(self, key):
835
"""See VersionedFiles.annotate."""
836
return self._factory.annotate(self, key)
838
def check(self, progress_bar=None):
839
"""See VersionedFiles.check()."""
840
# This doesn't actually test extraction of everything, but that will
841
# impact 'bzr check' substantially, and needs to be integrated with
842
# care. However, it does check for the obvious problem of a delta with
845
parent_map = self.get_parent_map(keys)
847
if self._index.get_method(key) != 'fulltext':
848
compression_parent = parent_map[key][0]
849
if compression_parent not in parent_map:
850
raise errors.KnitCorrupt(self,
851
"Missing basis parent %s for %s" % (
852
compression_parent, key))
854
def _check_add(self, key, lines, random_id, check_content):
855
"""check that version_id and lines are safe to add."""
856
if contains_whitespace(key[-1]):
857
raise InvalidRevisionId(key[-1], self.filename)
858
self.check_not_reserved_id(key[-1])
859
# Technically this could be avoided if we are happy to allow duplicate
860
# id insertion when other things than bzr core insert texts, but it
861
# seems useful for folk using the knit api directly to have some safety
862
# blanket that we can disable.
863
##if not random_id and self.has_version(key):
864
## raise RevisionAlreadyPresent(key, self)
866
self._check_lines_not_unicode(lines)
867
self._check_lines_are_lines(lines)
869
def _check_header(self, key, line):
870
rec = self._split_header(line)
871
self._check_header_version(rec, key[-1])
874
def _check_header_version(self, rec, version_id):
875
"""Checks the header version on original format knit records.
877
These have the last component of the key embedded in the record.
879
if rec[1] != version_id:
880
raise KnitCorrupt(self,
881
'unexpected version, wanted %r, got %r' % (version_id, rec[1]))
883
def _check_should_delta(self, parent):
884
"""Iterate back through the parent listing, looking for a fulltext.
886
This is used when we want to decide whether to add a delta or a new
887
fulltext. It searches for _max_delta_chain parents. When it finds a
888
fulltext parent, it sees if the total size of the deltas leading up to
889
it is large enough to indicate that we want a new full text anyway.
891
Return True if we should create a new delta, False if we should use a
896
for count in xrange(self._max_delta_chain):
897
# XXX: Collapse these two queries:
898
method = self._index.get_method(parent)
899
index, pos, size = self._index.get_position(parent)
900
if method == 'fulltext':
904
# No exception here because we stop at first fulltext anyway, an
905
# absent parent indicates a corrupt knit anyway.
906
# TODO: This should be asking for compression parent, not graph
908
parent = self._index.get_parent_map([parent])[parent][0]
910
# We couldn't find a fulltext, so we must create a new one
912
# Simple heuristic - if the total I/O wold be greater as a delta than
913
# the originally installed fulltext, we create a new fulltext.
914
return fulltext_size > delta_size
916
def _build_details_to_components(self, build_details):
917
"""Convert a build_details tuple to a position tuple."""
918
# record_details, access_memo, compression_parent
919
return build_details[3], build_details[0], build_details[1]
921
def _get_components_positions(self, keys, noraise=False):
922
"""Produce a map of position data for the components of keys.
924
This data is intended to be used for retrieving the knit records.
926
A dict of key to (record_details, index_memo, next, parents) is
928
method is the way referenced data should be applied.
929
index_memo is the handle to pass to the data access to actually get the
931
next is the build-parent of the version, or None for fulltexts.
932
parents is the version_ids of the parents of this version
934
:param noraise: If True do not raise an error on a missing component,
938
pending_components = keys
939
while pending_components:
940
build_details = self._index.get_build_details(pending_components)
941
current_components = set(pending_components)
942
pending_components = set()
943
for key, details in build_details.iteritems():
944
(index_memo, compression_parent, parents,
945
record_details) = details
946
method = record_details[0]
947
if compression_parent is not None:
948
pending_components.add(compression_parent)
949
component_data[key] = self._build_details_to_components(details)
950
missing = current_components.difference(build_details)
951
if missing and not noraise:
952
raise errors.RevisionNotPresent(missing.pop(), self)
953
return component_data
955
def _get_content(self, key, parent_texts={}):
956
"""Returns a content object that makes up the specified
958
cached_version = parent_texts.get(key, None)
959
if cached_version is not None:
960
# Ensure the cache dict is valid.
961
if not self.get_parent_map([key]):
962
raise RevisionNotPresent(key, self)
963
return cached_version
964
text_map, contents_map = self._get_content_maps([key])
965
return contents_map[key]
967
def _get_content_maps(self, keys):
968
"""Produce maps of text and KnitContents
970
:return: (text_map, content_map) where text_map contains the texts for
971
the requested versions and content_map contains the KnitContents.
973
# FUTURE: This function could be improved for the 'extract many' case
974
# by tracking each component and only doing the copy when the number of
975
# children than need to apply delta's to it is > 1 or it is part of the
978
multiple_versions = len(keys) != 1
979
record_map = self._get_record_map(keys)
987
while cursor is not None:
988
record, record_details, digest, next = record_map[cursor]
989
components.append((cursor, record, record_details, digest))
990
if cursor in content_map:
995
for (component_id, record, record_details,
996
digest) in reversed(components):
997
if component_id in content_map:
998
content = content_map[component_id]
1000
content, delta = self._factory.parse_record(key[-1],
1001
record, record_details, content,
1002
copy_base_content=multiple_versions)
1003
if multiple_versions:
1004
content_map[component_id] = content
1006
content.cleanup_eol(copy_on_mutate=multiple_versions)
1007
final_content[key] = content
1009
# digest here is the digest from the last applied component.
1010
text = content.text()
1011
actual_sha = sha_strings(text)
1012
if actual_sha != digest:
1013
raise KnitCorrupt(self,
1015
'\n of reconstructed text does not match'
1017
'\n for version %s' %
1018
(actual_sha, digest, key))
1019
text_map[key] = text
1020
return text_map, final_content
1022
def get_parent_map(self, keys):
1023
"""Get a map of the parents of keys.
1025
:param keys: The keys to look up parents for.
1026
:return: A mapping from keys to parents. Absent keys are absent from
1029
return self._index.get_parent_map(keys)
1031
def _get_record_map(self, keys):
1032
"""Produce a dictionary of knit records.
1034
:return: {key:(record, record_details, digest, next)}
1036
data returned from read_records
1038
opaque information to pass to parse_record
1040
SHA1 digest of the full text after all steps are done
1042
build-parent of the version, i.e. the leftmost ancestor.
1043
Will be None if the record is not a delta.
1045
position_map = self._get_components_positions(keys)
1046
# key = component_id, r = record_details, i_m = index_memo, n = next
1047
records = [(key, i_m) for key, (r, i_m, n)
1048
in position_map.iteritems()]
1050
for key, record, digest in \
1051
self._read_records_iter(records):
1052
(record_details, index_memo, next) = position_map[key]
1053
record_map[key] = record, record_details, digest, next
1056
def get_record_stream(self, keys, ordering, include_delta_closure):
1057
"""Get a stream of records for keys.
1059
:param keys: The keys to include.
1060
:param ordering: Either 'unordered' or 'topological'. A topologically
1061
sorted stream has compression parents strictly before their
1063
:param include_delta_closure: If True then the closure across any
1064
compression parents will be included (in the opaque data).
1065
:return: An iterator of ContentFactory objects, each of which is only
1066
valid until the iterator is advanced.
1068
# keys might be a generator
1070
if not self._index.has_graph:
1071
# Cannot topological order when no graph has been stored.
1072
ordering = 'unordered'
1073
if include_delta_closure:
1074
positions = self._get_components_positions(keys, noraise=True)
1076
build_details = self._index.get_build_details(keys)
1077
positions = dict((key, self._build_details_to_components(details))
1078
for key, details in build_details.iteritems())
1079
absent_keys = keys.difference(set(positions))
1080
# There may be more absent keys : if we're missing the basis component
1081
# and are trying to include the delta closure.
1082
if include_delta_closure:
1083
# key:True means key can be reconstructed
1088
chain = [key, positions[key][2]]
1090
absent_keys.add(key)
1093
while chain[-1] is not None:
1094
if chain[-1] in checked_keys:
1095
result = checked_keys[chain[-1]]
1099
chain.append(positions[chain[-1]][2])
1101
# missing basis component
1104
for chain_key in chain[:-1]:
1105
checked_keys[chain_key] = result
1107
absent_keys.add(key)
1108
for key in absent_keys:
1109
yield AbsentContentFactory(key)
1110
# restrict our view to the keys we can answer.
1111
keys = keys - absent_keys
1112
# Double index lookups here : need a unified api ?
1113
parent_map = self.get_parent_map(keys)
1114
if ordering == 'topological':
1115
present_keys = topo_sort(parent_map)
1118
# XXX: Memory: TODO: batch data here to cap buffered data at (say) 1MB.
1119
# XXX: At that point we need to consider double reads by utilising
1120
# components multiple times.
1121
if include_delta_closure:
1122
# XXX: get_content_maps performs its own index queries; allow state
1124
text_map, _ = self._get_content_maps(present_keys)
1125
for key in present_keys:
1126
yield FulltextContentFactory(key, parent_map[key], None,
1127
''.join(text_map[key]))
1129
records = [(key, positions[key][1]) for key in present_keys]
1130
for key, raw_data, sha1 in self._read_records_iter_raw(records):
1131
(record_details, index_memo, _) = positions[key]
1132
yield KnitContentFactory(key, parent_map[key],
1133
record_details, sha1, raw_data, self._factory.annotated, None)
1135
def get_sha1s(self, keys):
1136
"""See VersionedFiles.get_sha1s()."""
1137
record_map = self._get_record_map(keys)
1138
# record entry 2 is the 'digest'.
1139
return [record_map[key][2] for key in keys]
1141
def insert_record_stream(self, stream):
1142
"""Insert a record stream into this container.
1144
:param stream: A stream of records to insert.
1146
:seealso VersionedFiles.get_record_stream:
1148
def get_adapter(adapter_key):
1150
return adapters[adapter_key]
1152
adapter_factory = adapter_registry.get(adapter_key)
1153
adapter = adapter_factory(self)
1154
adapters[adapter_key] = adapter
1156
if self._factory.annotated:
1157
# self is annotated, we need annotated knits to use directly.
1158
annotated = "annotated-"
1161
# self is not annotated, but we can strip annotations cheaply.
1163
convertibles = set(["knit-annotated-ft-gz"])
1164
if self._max_delta_chain:
1165
convertibles.add("knit-annotated-delta-gz")
1166
# The set of types we can cheaply adapt without needing basis texts.
1167
native_types = set()
1168
if self._max_delta_chain:
1169
native_types.add("knit-%sdelta-gz" % annotated)
1170
native_types.add("knit-%sft-gz" % annotated)
1171
knit_types = native_types.union(convertibles)
1173
# Buffer all index entries that we can't add immediately because their
1174
# basis parent is missing. We don't buffer all because generating
1175
# annotations may require access to some of the new records. However we
1176
# can't generate annotations from new deltas until their basis parent
1177
# is present anyway, so we get away with not needing an index that
1178
# includes the new keys.
1179
# key = basis_parent, value = index entry to add
1180
buffered_index_entries = {}
1181
for record in stream:
1182
parents = record.parents
1183
# Raise an error when a record is missing.
1184
if record.storage_kind == 'absent':
1185
raise RevisionNotPresent([record.key], self)
1186
if record.storage_kind in knit_types:
1187
if record.storage_kind not in native_types:
1189
adapter_key = (record.storage_kind, "knit-delta-gz")
1190
adapter = get_adapter(adapter_key)
1192
adapter_key = (record.storage_kind, "knit-ft-gz")
1193
adapter = get_adapter(adapter_key)
1194
bytes = adapter.get_bytes(
1195
record, record.get_bytes_as(record.storage_kind))
1197
bytes = record.get_bytes_as(record.storage_kind)
1198
options = [record._build_details[0]]
1199
if record._build_details[1]:
1200
options.append('no-eol')
1201
# Just blat it across.
1202
# Note: This does end up adding data on duplicate keys. As
1203
# modern repositories use atomic insertions this should not
1204
# lead to excessive growth in the event of interrupted fetches.
1205
# 'knit' repositories may suffer excessive growth, but as a
1206
# deprecated format this is tolerable. It can be fixed if
1207
# needed by in the kndx index support raising on a duplicate
1208
# add with identical parents and options.
1209
access_memo = self._access.add_raw_records(
1210
[(record.key, len(bytes))], bytes)[0]
1211
index_entry = (record.key, options, access_memo, parents)
1213
if 'fulltext' not in options:
1214
basis_parent = parents[0]
1215
# Note that pack backed knits don't need to buffer here
1216
# because they buffer all writes to the transaction level,
1217
# but we don't expose that differnet at the index level. If
1218
# the query here has sufficient cost to show up in
1219
# profiling we should do that.
1220
if basis_parent not in self.get_parent_map([basis_parent]):
1221
pending = buffered_index_entries.setdefault(
1223
pending.append(index_entry)
1226
self._index.add_records([index_entry])
1227
elif record.storage_kind == 'fulltext':
1228
self.add_lines(record.key, parents,
1229
split_lines(record.get_bytes_as('fulltext')))
1231
adapter_key = record.storage_kind, 'fulltext'
1232
adapter = get_adapter(adapter_key)
1233
lines = split_lines(adapter.get_bytes(
1234
record, record.get_bytes_as(record.storage_kind)))
1236
self.add_lines(record.key, parents, lines)
1237
except errors.RevisionAlreadyPresent:
1239
# Add any records whose basis parent is now available.
1240
added_keys = [record.key]
1242
key = added_keys.pop(0)
1243
if key in buffered_index_entries:
1244
index_entries = buffered_index_entries[key]
1245
self._index.add_records(index_entries)
1247
[index_entry[0] for index_entry in index_entries])
1248
del buffered_index_entries[key]
1249
# If there were any deltas which had a missing basis parent, error.
1250
if buffered_index_entries:
1251
raise errors.RevisionNotPresent(buffered_index_entries.keys()[0],
1254
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1255
"""Iterate over the lines in the versioned files from keys.
1257
This may return lines from other keys. Each item the returned
1258
iterator yields is a tuple of a line and a text version that that line
1259
is present in (not introduced in).
1261
Ordering of results is in whatever order is most suitable for the
1262
underlying storage format.
1264
If a progress bar is supplied, it may be used to indicate progress.
1265
The caller is responsible for cleaning up progress bars (because this
1269
* Lines are normalised by the underlying store: they will all have \n
1271
* Lines are returned in arbitrary order.
1273
:return: An iterator over (line, key).
1276
pb = progress.DummyProgress()
1278
# filter for available keys
1279
parent_map = self.get_parent_map(keys)
1280
if len(parent_map) != len(keys):
1281
missing = set(parent_map) - requested_keys
1282
raise RevisionNotPresent(key, self.filename)
1283
# we don't care about inclusions, the caller cares.
1284
# but we need to setup a list of records to visit.
1285
# we need key, position, length
1287
build_details = self._index.get_build_details(keys)
1289
key_records.append((key, build_details[key][0]))
1290
total = len(key_records)
1291
for key_idx, (key, data, sha_value) in \
1292
enumerate(self._read_records_iter(key_records)):
1293
pb.update('Walking content.', key_idx, total)
1294
compression_parent = build_details[key][1]
1295
if compression_parent is None:
1297
line_iterator = self._factory.get_fulltext_content(data)
1300
line_iterator = self._factory.get_linedelta_content(data)
1301
# XXX: It might be more efficient to yield (key,
1302
# line_iterator) in the future. However for now, this is a simpler
1303
# change to integrate into the rest of the codebase. RBC 20071110
1304
for line in line_iterator:
1306
pb.update('Walking content.', total, total)
1308
def _make_line_delta(self, delta_seq, new_content):
1309
"""Generate a line delta from delta_seq and new_content."""
1311
for op in delta_seq.get_opcodes():
1312
if op[0] == 'equal':
1314
diff_hunks.append((op[1], op[2], op[4]-op[3], new_content._lines[op[3]:op[4]]))
1317
def _merge_annotations(self, content, parents, parent_texts={},
1318
delta=None, annotated=None,
1319
left_matching_blocks=None):
1320
"""Merge annotations for content and generate deltas.
1322
This is done by comparing the annotations based on changes to the text
1323
and generating a delta on the resulting full texts. If annotations are
1324
not being created then a simple delta is created.
1326
if left_matching_blocks is not None:
1327
delta_seq = diff._PrematchedMatcher(left_matching_blocks)
1331
for parent_key in parents:
1332
merge_content = self._get_content(parent_key, parent_texts)
1333
if (parent_key == parents[0] and delta_seq is not None):
1336
seq = patiencediff.PatienceSequenceMatcher(
1337
None, merge_content.text(), content.text())
1338
for i, j, n in seq.get_matching_blocks():
1341
# this appears to copy (origin, text) pairs across to the
1342
# new content for any line that matches the last-checked
1344
content._lines[j:j+n] = merge_content._lines[i:i+n]
1345
if content._lines and content._lines[-1][1][-1] != '\n':
1346
# The copied annotation was from a line without a trailing EOL,
1347
# reinstate one for the content object, to ensure correct
1349
line = content._lines[-1][1] + '\n'
1350
content._lines[-1] = (content._lines[-1][0], line)
1352
if delta_seq is None:
1353
reference_content = self._get_content(parents[0], parent_texts)
1354
new_texts = content.text()
1355
old_texts = reference_content.text()
1356
delta_seq = patiencediff.PatienceSequenceMatcher(
1357
None, old_texts, new_texts)
1358
return self._make_line_delta(delta_seq, content)
1360
def _parse_record(self, version_id, data):
1361
"""Parse an original format knit record.
1363
These have the last element of the key only present in the stored data.
1365
rec, record_contents = self._parse_record_unchecked(data)
1366
self._check_header_version(rec, version_id)
1367
return record_contents, rec[3]
1369
def _parse_record_header(self, key, raw_data):
1370
"""Parse a record header for consistency.
1372
:return: the header and the decompressor stream.
1373
as (stream, header_record)
1375
df = GzipFile(mode='rb', fileobj=StringIO(raw_data))
1378
rec = self._check_header(key, df.readline())
1379
except Exception, e:
1380
raise KnitCorrupt(self,
1381
"While reading {%s} got %s(%s)"
1382
% (key, e.__class__.__name__, str(e)))
1385
def _parse_record_unchecked(self, data):
1387
# 4168 calls in 2880 217 internal
1388
# 4168 calls to _parse_record_header in 2121
1389
# 4168 calls to readlines in 330
1390
df = GzipFile(mode='rb', fileobj=StringIO(data))
1392
record_contents = df.readlines()
1393
except Exception, e:
1394
raise KnitCorrupt(self, "Corrupt compressed record %r, got %s(%s)" %
1395
(data, e.__class__.__name__, str(e)))
1396
header = record_contents.pop(0)
1397
rec = self._split_header(header)
1398
last_line = record_contents.pop()
1399
if len(record_contents) != int(rec[2]):
1400
raise KnitCorrupt(self,
1401
'incorrect number of lines %s != %s'
1402
' for version {%s} %s'
1403
% (len(record_contents), int(rec[2]),
1404
rec[1], record_contents))
1405
if last_line != 'end %s\n' % rec[1]:
1406
raise KnitCorrupt(self,
1407
'unexpected version end line %r, wanted %r'
1408
% (last_line, rec[1]))
1410
return rec, record_contents
1412
def _read_records_iter(self, records):
1413
"""Read text records from data file and yield result.
1415
The result will be returned in whatever is the fastest to read.
1416
Not by the order requested. Also, multiple requests for the same
1417
record will only yield 1 response.
1418
:param records: A list of (key, access_memo) entries
1419
:return: Yields (key, contents, digest) in the order
1420
read, not the order requested
1425
# XXX: This smells wrong, IO may not be getting ordered right.
1426
needed_records = sorted(set(records), key=operator.itemgetter(1))
1427
if not needed_records:
1430
# The transport optimizes the fetching as well
1431
# (ie, reads continuous ranges.)
1432
raw_data = self._access.get_raw_records(
1433
[index_memo for key, index_memo in needed_records])
1435
for (key, index_memo), data in \
1436
izip(iter(needed_records), raw_data):
1437
content, digest = self._parse_record(key[-1], data)
1438
yield key, content, digest
1440
def _read_records_iter_raw(self, records):
1441
"""Read text records from data file and yield raw data.
1443
This unpacks enough of the text record to validate the id is
1444
as expected but thats all.
1446
Each item the iterator yields is (key, bytes, sha1_of_full_text).
1448
# setup an iterator of the external records:
1449
# uses readv so nice and fast we hope.
1451
# grab the disk data needed.
1452
needed_offsets = [index_memo for key, index_memo
1454
raw_records = self._access.get_raw_records(needed_offsets)
1456
for key, index_memo in records:
1457
data = raw_records.next()
1458
# validate the header (note that we can only use the suffix in
1459
# current knit records).
1460
df, rec = self._parse_record_header(key, data)
1462
yield key, data, rec[3]
1464
def _record_to_data(self, key, digest, lines, dense_lines=None):
1465
"""Convert key, digest, lines into a raw data block.
1467
:param key: The key of the record. Currently keys are always serialised
1468
using just the trailing component.
1469
:param dense_lines: The bytes of lines but in a denser form. For
1470
instance, if lines is a list of 1000 bytestrings each ending in \n,
1471
dense_lines may be a list with one line in it, containing all the
1472
1000's lines and their \n's. Using dense_lines if it is already
1473
known is a win because the string join to create bytes in this
1474
function spends less time resizing the final string.
1475
:return: (len, a StringIO instance with the raw data ready to read.)
1477
# Note: using a string copy here increases memory pressure with e.g.
1478
# ISO's, but it is about 3 seconds faster on a 1.2Ghz intel machine
1479
# when doing the initial commit of a mozilla tree. RBC 20070921
1480
bytes = ''.join(chain(
1481
["version %s %d %s\n" % (key[-1],
1484
dense_lines or lines,
1485
["end %s\n" % key[-1]]))
1486
if type(bytes) != str:
1487
raise AssertionError(
1488
'data must be plain bytes was %s' % type(bytes))
1489
if lines and lines[-1][-1] != '\n':
1490
raise ValueError('corrupt lines value %r' % lines)
1491
compressed_bytes = bytes_to_gzip(bytes)
1492
return len(compressed_bytes), compressed_bytes
1494
def _split_header(self, line):
1497
raise KnitCorrupt(self,
1498
'unexpected number of elements in record header')
1502
"""See VersionedFiles.keys."""
1503
if 'evil' in debug.debug_flags:
1504
trace.mutter_callsite(2, "keys scales with size of history")
1505
return self._index.keys()
1508
class _KndxIndex(object):
1509
"""Manages knit index files
1511
The index is kept in memorya already kept in memory and read on startup, to enable
1512
fast lookups of revision information. The cursor of the index
1513
file is always pointing to the end, making it easy to append
1516
_cache is a cache for fast mapping from version id to a Index
1519
_history is a cache for fast mapping from indexes to version ids.
1521
The index data format is dictionary compressed when it comes to
1522
parent references; a index entry may only have parents that with a
1523
lover index number. As a result, the index is topological sorted.
1525
Duplicate entries may be written to the index for a single version id
1526
if this is done then the latter one completely replaces the former:
1527
this allows updates to correct version and parent information.
1528
Note that the two entries may share the delta, and that successive
1529
annotations and references MUST point to the first entry.
1531
The index file on disc contains a header, followed by one line per knit
1532
record. The same revision can be present in an index file more than once.
1533
The first occurrence gets assigned a sequence number starting from 0.
1535
The format of a single line is
1536
REVISION_ID FLAGS BYTE_OFFSET LENGTH( PARENT_ID|PARENT_SEQUENCE_ID)* :\n
1537
REVISION_ID is a utf8-encoded revision id
1538
FLAGS is a comma separated list of flags about the record. Values include
1539
no-eol, line-delta, fulltext.
1540
BYTE_OFFSET is the ascii representation of the byte offset in the data file
1541
that the the compressed data starts at.
1542
LENGTH is the ascii representation of the length of the data file.
1543
PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
1545
PARENT_SEQUENCE_ID the ascii representation of the sequence number of a
1546
revision id already in the knit that is a parent of REVISION_ID.
1547
The ' :' marker is the end of record marker.
1550
when a write is interrupted to the index file, it will result in a line
1551
that does not end in ' :'. If the ' :' is not present at the end of a line,
1552
or at the end of the file, then the record that is missing it will be
1553
ignored by the parser.
1555
When writing new records to the index file, the data is preceded by '\n'
1556
to ensure that records always start on new lines even if the last write was
1557
interrupted. As a result its normal for the last line in the index to be
1558
missing a trailing newline. One can be added with no harmful effects.
1561
HEADER = "# bzr knit index 8\n"
1563
def __init__(self, transport, mapper, get_scope, allow_writes, is_locked):
1564
"""Create a _KndxIndex on transport using mapper."""
1565
self._transport = transport
1566
self._mapper = mapper
1567
self._get_scope = get_scope
1568
self._allow_writes = allow_writes
1569
self._is_locked = is_locked
1571
self.has_graph = True
1573
def add_records(self, records, random_id=False):
1574
"""Add multiple records to the index.
1576
:param records: a list of tuples:
1577
(key, options, access_memo, parents).
1578
:param random_id: If True the ids being added were randomly generated
1579
and no check for existence will be performed.
1582
for record in records:
1585
path = self._mapper.map(key) + '.kndx'
1586
path_keys = paths.setdefault(path, (prefix, []))
1587
path_keys[1].append(record)
1588
for path in sorted(paths):
1589
prefix, path_keys = paths[path]
1590
self._load_prefixes([prefix])
1592
orig_history = self._kndx_cache[prefix][1][:]
1593
orig_cache = self._kndx_cache[prefix][0].copy()
1596
for key, options, (_, pos, size), parents in path_keys:
1598
# kndx indices cannot be parentless.
1600
line = "\n%s %s %s %s %s :" % (
1601
key[-1], ','.join(options), pos, size,
1602
self._dictionary_compress(parents))
1603
if type(line) != str:
1604
raise AssertionError(
1605
'data must be utf8 was %s' % type(line))
1607
self._cache_key(key, options, pos, size, parents)
1608
if len(orig_history):
1609
self._transport.append_bytes(path, ''.join(lines))
1611
self._init_index(path, lines)
1613
# If any problems happen, restore the original values and re-raise
1614
self._kndx_cache[prefix] = (orig_cache, orig_history)
1617
def _cache_key(self, key, options, pos, size, parent_keys):
1618
"""Cache a version record in the history array and index cache.
1620
This is inlined into _load_data for performance. KEEP IN SYNC.
1621
(It saves 60ms, 25% of the __init__ overhead on local 4000 record
1625
version_id = key[-1]
1626
# last-element only for compatibilty with the C load_data.
1627
parents = tuple(parent[-1] for parent in parent_keys)
1628
for parent in parent_keys:
1629
if parent[:-1] != prefix:
1630
raise ValueError("mismatched prefixes for %r, %r" % (
1632
cache, history = self._kndx_cache[prefix]
1633
# only want the _history index to reference the 1st index entry
1635
if version_id not in cache:
1636
index = len(history)
1637
history.append(version_id)
1639
index = cache[version_id][5]
1640
cache[version_id] = (version_id,
1647
def check_header(self, fp):
1648
line = fp.readline()
1650
# An empty file can actually be treated as though the file doesn't
1652
raise errors.NoSuchFile(self)
1653
if line != self.HEADER:
1654
raise KnitHeaderError(badline=line, filename=self)
1656
def _check_read(self):
1657
if not self._is_locked():
1658
raise errors.ObjectNotLocked(self)
1659
if self._get_scope() != self._scope:
1662
def _check_write_ok(self):
1663
"""Assert if not writes are permitted."""
1664
if not self._is_locked():
1665
raise errors.ObjectNotLocked(self)
1666
if self._get_scope() != self._scope:
1668
if self._mode != 'w':
1669
raise errors.ReadOnlyObjectDirtiedError(self)
1671
def get_build_details(self, keys):
1672
"""Get the method, index_memo and compression parent for keys.
1674
Ghosts are omitted from the result.
1676
:param keys: An iterable of keys.
1677
:return: A dict of key:(access_memo, compression_parent, parents,
1680
opaque structure to pass to read_records to extract the raw
1683
Content that this record is built upon, may be None
1685
Logical parents of this node
1687
extra information about the content which needs to be passed to
1688
Factory.parse_record
1690
prefixes = self._partition_keys(keys)
1691
parent_map = self.get_parent_map(keys)
1694
if key not in parent_map:
1696
method = self.get_method(key)
1697
parents = parent_map[key]
1698
if method == 'fulltext':
1699
compression_parent = None
1701
compression_parent = parents[0]
1702
noeol = 'no-eol' in self.get_options(key)
1703
index_memo = self.get_position(key)
1704
result[key] = (index_memo, compression_parent,
1705
parents, (method, noeol))
1708
def get_method(self, key):
1709
"""Return compression method of specified key."""
1710
options = self.get_options(key)
1711
if 'fulltext' in options:
1713
elif 'line-delta' in options:
1716
raise errors.KnitIndexUnknownMethod(self, options)
1718
def get_options(self, key):
1719
"""Return a list representing options.
1723
prefix, suffix = self._split_key(key)
1724
self._load_prefixes([prefix])
1725
return self._kndx_cache[prefix][0][suffix][1]
1727
def get_parent_map(self, keys):
1728
"""Get a map of the parents of keys.
1730
:param keys: The keys to look up parents for.
1731
:return: A mapping from keys to parents. Absent keys are absent from
1734
# Parse what we need to up front, this potentially trades off I/O
1735
# locality (.kndx and .knit in the same block group for the same file
1736
# id) for less checking in inner loops.
1738
prefixes.update(key[:-1] for key in keys)
1739
self._load_prefixes(prefixes)
1744
suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
1748
result[key] = tuple(prefix + (suffix,) for
1749
suffix in suffix_parents)
1752
def get_position(self, key):
1753
"""Return details needed to access the version.
1755
:return: a tuple (key, data position, size) to hand to the access
1756
logic to get the record.
1758
prefix, suffix = self._split_key(key)
1759
self._load_prefixes([prefix])
1760
entry = self._kndx_cache[prefix][0][suffix]
1761
return key, entry[2], entry[3]
1763
def _init_index(self, path, extra_lines=[]):
1764
"""Initialize an index."""
1766
sio.write(self.HEADER)
1767
sio.writelines(extra_lines)
1769
self._transport.put_file_non_atomic(path, sio,
1770
create_parent_dir=True)
1771
# self._create_parent_dir)
1772
# mode=self._file_mode,
1773
# dir_mode=self._dir_mode)
1776
"""Get all the keys in the collection.
1778
The keys are not ordered.
1781
# Identify all key prefixes.
1782
# XXX: A bit hacky, needs polish.
1783
if type(self._mapper) == ConstantMapper:
1787
for quoted_relpath in self._transport.iter_files_recursive():
1788
path, ext = os.path.splitext(quoted_relpath)
1790
prefixes = [self._mapper.unmap(path) for path in relpaths]
1791
self._load_prefixes(prefixes)
1792
for prefix in prefixes:
1793
for suffix in self._kndx_cache[prefix][1]:
1794
result.add(prefix + (suffix,))
1797
def _load_prefixes(self, prefixes):
1798
"""Load the indices for prefixes."""
1800
for prefix in prefixes:
1801
if prefix not in self._kndx_cache:
1802
# the load_data interface writes to these variables.
1805
self._filename = prefix
1807
path = self._mapper.map(prefix) + '.kndx'
1808
fp = self._transport.get(path)
1810
# _load_data may raise NoSuchFile if the target knit is
1812
_load_data(self, fp)
1815
self._kndx_cache[prefix] = (self._cache, self._history)
1820
self._kndx_cache[prefix] = ({}, [])
1821
if type(self._mapper) == ConstantMapper:
1822
# preserve behaviour for revisions.kndx etc.
1823
self._init_index(path)
1828
def _partition_keys(self, keys):
1829
"""Turn keys into a dict of prefix:suffix_list."""
1832
prefix_keys = result.setdefault(key[:-1], [])
1833
prefix_keys.append(key[-1])
1836
def _dictionary_compress(self, keys):
1837
"""Dictionary compress keys.
1839
:param keys: The keys to generate references to.
1840
:return: A string representation of keys. keys which are present are
1841
dictionary compressed, and others are emitted as fulltext with a
1847
prefix = keys[0][:-1]
1848
cache = self._kndx_cache[prefix][0]
1850
if key[:-1] != prefix:
1851
# kndx indices cannot refer across partitioned storage.
1852
raise ValueError("mismatched prefixes for %r" % keys)
1853
if key[-1] in cache:
1854
# -- inlined lookup() --
1855
result_list.append(str(cache[key[-1]][5]))
1856
# -- end lookup () --
1858
result_list.append('.' + key[-1])
1859
return ' '.join(result_list)
1861
def _reset_cache(self):
1862
# Possibly this should be a LRU cache. A dictionary from key_prefix to
1863
# (cache_dict, history_vector) for parsed kndx files.
1864
self._kndx_cache = {}
1865
self._scope = self._get_scope()
1866
allow_writes = self._allow_writes()
1872
def _split_key(self, key):
1873
"""Split key into a prefix and suffix."""
1874
return key[:-1], key[-1]
1877
class _KnitGraphIndex(object):
1878
"""A KnitVersionedFiles index layered on GraphIndex."""
1880
def __init__(self, graph_index, is_locked, deltas=False, parents=True,
1882
"""Construct a KnitGraphIndex on a graph_index.
1884
:param graph_index: An implementation of bzrlib.index.GraphIndex.
1885
:param is_locked: A callback to check whether the object should answer
1887
:param deltas: Allow delta-compressed records.
1888
:param parents: If True, record knits parents, if not do not record
1890
:param add_callback: If not None, allow additions to the index and call
1891
this callback with a list of added GraphIndex nodes:
1892
[(node, value, node_refs), ...]
1893
:param is_locked: A callback, returns True if the index is locked and
1896
self._add_callback = add_callback
1897
self._graph_index = graph_index
1898
self._deltas = deltas
1899
self._parents = parents
1900
if deltas and not parents:
1901
# XXX: TODO: Delta tree and parent graph should be conceptually
1903
raise KnitCorrupt(self, "Cannot do delta compression without "
1905
self.has_graph = parents
1906
self._is_locked = is_locked
1908
def add_records(self, records, random_id=False):
1909
"""Add multiple records to the index.
1911
This function does not insert data into the Immutable GraphIndex
1912
backing the KnitGraphIndex, instead it prepares data for insertion by
1913
the caller and checks that it is safe to insert then calls
1914
self._add_callback with the prepared GraphIndex nodes.
1916
:param records: a list of tuples:
1917
(key, options, access_memo, parents).
1918
:param random_id: If True the ids being added were randomly generated
1919
and no check for existence will be performed.
1921
if not self._add_callback:
1922
raise errors.ReadOnlyError(self)
1923
# we hope there are no repositories with inconsistent parentage
1927
for (key, options, access_memo, parents) in records:
1929
parents = tuple(parents)
1930
index, pos, size = access_memo
1931
if 'no-eol' in options:
1935
value += "%d %d" % (pos, size)
1936
if not self._deltas:
1937
if 'line-delta' in options:
1938
raise KnitCorrupt(self, "attempt to add line-delta in non-delta knit")
1941
if 'line-delta' in options:
1942
node_refs = (parents, (parents[0],))
1944
node_refs = (parents, ())
1946
node_refs = (parents, )
1949
raise KnitCorrupt(self, "attempt to add node with parents "
1950
"in parentless index.")
1952
keys[key] = (value, node_refs)
1955
present_nodes = self._get_entries(keys)
1956
for (index, key, value, node_refs) in present_nodes:
1957
if (value[0] != keys[key][0][0] or
1958
node_refs != keys[key][1]):
1959
raise KnitCorrupt(self, "inconsistent details in add_records"
1960
": %s %s" % ((value, node_refs), keys[key]))
1964
for key, (value, node_refs) in keys.iteritems():
1965
result.append((key, value, node_refs))
1967
for key, (value, node_refs) in keys.iteritems():
1968
result.append((key, value))
1969
self._add_callback(result)
1971
def _check_read(self):
1972
"""raise if reads are not permitted."""
1973
if not self._is_locked():
1974
raise errors.ObjectNotLocked(self)
1976
def _check_write_ok(self):
1977
"""Assert if writes are not permitted."""
1978
if not self._is_locked():
1979
raise errors.ObjectNotLocked(self)
1981
def _compression_parent(self, an_entry):
1982
# return the key that an_entry is compressed against, or None
1983
# Grab the second parent list (as deltas implies parents currently)
1984
compression_parents = an_entry[3][1]
1985
if not compression_parents:
1987
if len(compression_parents) != 1:
1988
raise AssertionError(
1989
"Too many compression parents: %r" % compression_parents)
1990
return compression_parents[0]
1992
def get_build_details(self, keys):
1993
"""Get the method, index_memo and compression parent for version_ids.
1995
Ghosts are omitted from the result.
1997
:param keys: An iterable of keys.
1998
:return: A dict of key:
1999
(index_memo, compression_parent, parents, record_details).
2001
opaque structure to pass to read_records to extract the raw
2004
Content that this record is built upon, may be None
2006
Logical parents of this node
2008
extra information about the content which needs to be passed to
2009
Factory.parse_record
2013
entries = self._get_entries(keys, False)
2014
for entry in entries:
2016
if not self._parents:
2019
parents = entry[3][0]
2020
if not self._deltas:
2021
compression_parent_key = None
2023
compression_parent_key = self._compression_parent(entry)
2024
noeol = (entry[2][0] == 'N')
2025
if compression_parent_key:
2026
method = 'line-delta'
2029
result[key] = (self._node_to_position(entry),
2030
compression_parent_key, parents,
2034
def _get_entries(self, keys, check_present=False):
2035
"""Get the entries for keys.
2037
:param keys: An iterable of index key tuples.
2042
for node in self._graph_index.iter_entries(keys):
2044
found_keys.add(node[1])
2046
# adapt parentless index to the rest of the code.
2047
for node in self._graph_index.iter_entries(keys):
2048
yield node[0], node[1], node[2], ()
2049
found_keys.add(node[1])
2051
missing_keys = keys.difference(found_keys)
2053
raise RevisionNotPresent(missing_keys.pop(), self)
2055
def get_method(self, key):
2056
"""Return compression method of specified key."""
2057
return self._get_method(self._get_node(key))
2059
def _get_method(self, node):
2060
if not self._deltas:
2062
if self._compression_parent(node):
2067
def _get_node(self, key):
2069
return list(self._get_entries([key]))[0]
2071
raise RevisionNotPresent(key, self)
2073
def get_options(self, key):
2074
"""Return a list representing options.
2078
node = self._get_node(key)
2079
options = [self._get_method(node)]
2080
if node[2][0] == 'N':
2081
options.append('no-eol')
2084
def get_parent_map(self, keys):
2085
"""Get a map of the parents of keys.
2087
:param keys: The keys to look up parents for.
2088
:return: A mapping from keys to parents. Absent keys are absent from
2092
nodes = self._get_entries(keys)
2096
result[node[1]] = node[3][0]
2099
result[node[1]] = None
2102
def get_position(self, key):
2103
"""Return details needed to access the version.
2105
:return: a tuple (index, data position, size) to hand to the access
2106
logic to get the record.
2108
node = self._get_node(key)
2109
return self._node_to_position(node)
2112
"""Get all the keys in the collection.
2114
The keys are not ordered.
2117
return [node[1] for node in self._graph_index.iter_all_entries()]
2119
def _node_to_position(self, node):
2120
"""Convert an index value to position details."""
2121
bits = node[2][1:].split(' ')
2122
return node[0], int(bits[0]), int(bits[1])
2125
class _KnitKeyAccess(object):
2126
"""Access to records in .knit files."""
2128
def __init__(self, transport, mapper):
2129
"""Create a _KnitKeyAccess with transport and mapper.
2131
:param transport: The transport the access object is rooted at.
2132
:param mapper: The mapper used to map keys to .knit files.
2134
self._transport = transport
2135
self._mapper = mapper
2137
def add_raw_records(self, key_sizes, raw_data):
2138
"""Add raw knit bytes to a storage area.
2140
The data is spooled to the container writer in one bytes-record per
2143
:param sizes: An iterable of tuples containing the key and size of each
2145
:param raw_data: A bytestring containing the data.
2146
:return: A list of memos to retrieve the record later. Each memo is an
2147
opaque index memo. For _KnitKeyAccess the memo is (key, pos,
2148
length), where the key is the record key.
2150
if type(raw_data) != str:
2151
raise AssertionError(
2152
'data must be plain bytes was %s' % type(raw_data))
2155
# TODO: This can be tuned for writing to sftp and other servers where
2156
# append() is relatively expensive by grouping the writes to each key
2158
for key, size in key_sizes:
2159
path = self._mapper.map(key)
2161
base = self._transport.append_bytes(path + '.knit',
2162
raw_data[offset:offset+size])
2163
except errors.NoSuchFile:
2164
self._transport.mkdir(osutils.dirname(path))
2165
base = self._transport.append_bytes(path + '.knit',
2166
raw_data[offset:offset+size])
2170
result.append((key, base, size))
2173
def get_raw_records(self, memos_for_retrieval):
2174
"""Get the raw bytes for a records.
2176
:param memos_for_retrieval: An iterable containing the access memo for
2177
retrieving the bytes.
2178
:return: An iterator over the bytes of the records.
2180
# first pass, group into same-index request to minimise readv's issued.
2182
current_prefix = None
2183
for (key, offset, length) in memos_for_retrieval:
2184
if current_prefix == key[:-1]:
2185
current_list.append((offset, length))
2187
if current_prefix is not None:
2188
request_lists.append((current_prefix, current_list))
2189
current_prefix = key[:-1]
2190
current_list = [(offset, length)]
2191
# handle the last entry
2192
if current_prefix is not None:
2193
request_lists.append((current_prefix, current_list))
2194
for prefix, read_vector in request_lists:
2195
path = self._mapper.map(prefix) + '.knit'
2196
for pos, data in self._transport.readv(path, read_vector):
2200
class _DirectPackAccess(object):
2201
"""Access to data in one or more packs with less translation."""
2203
def __init__(self, index_to_packs):
2204
"""Create a _DirectPackAccess object.
2206
:param index_to_packs: A dict mapping index objects to the transport
2207
and file names for obtaining data.
2209
self._container_writer = None
2210
self._write_index = None
2211
self._indices = index_to_packs
2213
def add_raw_records(self, key_sizes, raw_data):
2214
"""Add raw knit bytes to a storage area.
2216
The data is spooled to the container writer in one bytes-record per
2219
:param sizes: An iterable of tuples containing the key and size of each
2221
:param raw_data: A bytestring containing the data.
2222
:return: A list of memos to retrieve the record later. Each memo is an
2223
opaque index memo. For _DirectPackAccess the memo is (index, pos,
2224
length), where the index field is the write_index object supplied
2225
to the PackAccess object.
2227
if type(raw_data) != str:
2228
raise AssertionError(
2229
'data must be plain bytes was %s' % type(raw_data))
2232
for key, size in key_sizes:
2233
p_offset, p_length = self._container_writer.add_bytes_record(
2234
raw_data[offset:offset+size], [])
2236
result.append((self._write_index, p_offset, p_length))
2239
def get_raw_records(self, memos_for_retrieval):
2240
"""Get the raw bytes for a records.
2242
:param memos_for_retrieval: An iterable containing the (index, pos,
2243
length) memo for retrieving the bytes. The Pack access method
2244
looks up the pack to use for a given record in its index_to_pack
2246
:return: An iterator over the bytes of the records.
2248
# first pass, group into same-index requests
2250
current_index = None
2251
for (index, offset, length) in memos_for_retrieval:
2252
if current_index == index:
2253
current_list.append((offset, length))
2255
if current_index is not None:
2256
request_lists.append((current_index, current_list))
2257
current_index = index
2258
current_list = [(offset, length)]
2259
# handle the last entry
2260
if current_index is not None:
2261
request_lists.append((current_index, current_list))
2262
for index, offsets in request_lists:
2263
transport, path = self._indices[index]
2264
reader = pack.make_readv_reader(transport, path, offsets)
2265
for names, read_func in reader.iter_records():
2266
yield read_func(None)
2268
def set_writer(self, writer, index, transport_packname):
2269
"""Set a writer to use for adding data."""
2270
if index is not None:
2271
self._indices[index] = transport_packname
2272
self._container_writer = writer
2273
self._write_index = index
2276
# Deprecated, use PatienceSequenceMatcher instead
2277
KnitSequenceMatcher = patiencediff.PatienceSequenceMatcher
2280
def annotate_knit(knit, revision_id):
2281
"""Annotate a knit with no cached annotations.
2283
This implementation is for knits with no cached annotations.
2284
It will work for knits with cached annotations, but this is not
2287
annotator = _KnitAnnotator(knit)
2288
return iter(annotator.annotate(revision_id))
2291
class _KnitAnnotator(object):
2292
"""Build up the annotations for a text."""
2294
def __init__(self, knit):
2297
# Content objects, differs from fulltexts because of how final newlines
2298
# are treated by knits. the content objects here will always have a
2300
self._fulltext_contents = {}
2302
# Annotated lines of specific revisions
2303
self._annotated_lines = {}
2305
# Track the raw data for nodes that we could not process yet.
2306
# This maps the revision_id of the base to a list of children that will
2307
# annotated from it.
2308
self._pending_children = {}
2310
# Nodes which cannot be extracted
2311
self._ghosts = set()
2313
# Track how many children this node has, so we know if we need to keep
2315
self._annotate_children = {}
2316
self._compression_children = {}
2318
self._all_build_details = {}
2319
# The children => parent revision_id graph
2320
self._revision_id_graph = {}
2322
self._heads_provider = None
2324
self._nodes_to_keep_annotations = set()
2325
self._generations_until_keep = 100
2327
def set_generations_until_keep(self, value):
2328
"""Set the number of generations before caching a node.
2330
Setting this to -1 will cache every merge node, setting this higher
2331
will cache fewer nodes.
2333
self._generations_until_keep = value
2335
def _add_fulltext_content(self, revision_id, content_obj):
2336
self._fulltext_contents[revision_id] = content_obj
2337
# TODO: jam 20080305 It might be good to check the sha1digest here
2338
return content_obj.text()
2340
def _check_parents(self, child, nodes_to_annotate):
2341
"""Check if all parents have been processed.
2343
:param child: A tuple of (rev_id, parents, raw_content)
2344
:param nodes_to_annotate: If child is ready, add it to
2345
nodes_to_annotate, otherwise put it back in self._pending_children
2347
for parent_id in child[1]:
2348
if (parent_id not in self._annotated_lines):
2349
# This parent is present, but another parent is missing
2350
self._pending_children.setdefault(parent_id,
2354
# This one is ready to be processed
2355
nodes_to_annotate.append(child)
2357
def _add_annotation(self, revision_id, fulltext, parent_ids,
2358
left_matching_blocks=None):
2359
"""Add an annotation entry.
2361
All parents should already have been annotated.
2362
:return: A list of children that now have their parents satisfied.
2364
a = self._annotated_lines
2365
annotated_parent_lines = [a[p] for p in parent_ids]
2366
annotated_lines = list(annotate.reannotate(annotated_parent_lines,
2367
fulltext, revision_id, left_matching_blocks,
2368
heads_provider=self._get_heads_provider()))
2369
self._annotated_lines[revision_id] = annotated_lines
2370
for p in parent_ids:
2371
ann_children = self._annotate_children[p]
2372
ann_children.remove(revision_id)
2373
if (not ann_children
2374
and p not in self._nodes_to_keep_annotations):
2375
del self._annotated_lines[p]
2376
del self._all_build_details[p]
2377
if p in self._fulltext_contents:
2378
del self._fulltext_contents[p]
2379
# Now that we've added this one, see if there are any pending
2380
# deltas to be done, certainly this parent is finished
2381
nodes_to_annotate = []
2382
for child in self._pending_children.pop(revision_id, []):
2383
self._check_parents(child, nodes_to_annotate)
2384
return nodes_to_annotate
2386
def _get_build_graph(self, key):
2387
"""Get the graphs for building texts and annotations.
2389
The data you need for creating a full text may be different than the
2390
data you need to annotate that text. (At a minimum, you need both
2391
parents to create an annotation, but only need 1 parent to generate the
2394
:return: A list of (key, index_memo) records, suitable for
2395
passing to read_records_iter to start reading in the raw data fro/
2398
if key in self._annotated_lines:
2401
pending = set([key])
2406
# get all pending nodes
2408
this_iteration = pending
2409
build_details = self._knit._index.get_build_details(this_iteration)
2410
self._all_build_details.update(build_details)
2411
# new_nodes = self._knit._index._get_entries(this_iteration)
2413
for key, details in build_details.iteritems():
2414
(index_memo, compression_parent, parents,
2415
record_details) = details
2416
self._revision_id_graph[key] = parents
2417
records.append((key, index_memo))
2418
# Do we actually need to check _annotated_lines?
2419
pending.update(p for p in parents
2420
if p not in self._all_build_details)
2421
if compression_parent:
2422
self._compression_children.setdefault(compression_parent,
2425
for parent in parents:
2426
self._annotate_children.setdefault(parent,
2428
num_gens = generation - kept_generation
2429
if ((num_gens >= self._generations_until_keep)
2430
and len(parents) > 1):
2431
kept_generation = generation
2432
self._nodes_to_keep_annotations.add(key)
2434
missing_versions = this_iteration.difference(build_details.keys())
2435
self._ghosts.update(missing_versions)
2436
for missing_version in missing_versions:
2437
# add a key, no parents
2438
self._revision_id_graph[missing_version] = ()
2439
pending.discard(missing_version) # don't look for it
2440
if self._ghosts.intersection(self._compression_children):
2442
"We cannot have nodes which have a ghost compression parent:\n"
2444
"compression children: %r"
2445
% (self._ghosts, self._compression_children))
2446
# Cleanout anything that depends on a ghost so that we don't wait for
2447
# the ghost to show up
2448
for node in self._ghosts:
2449
if node in self._annotate_children:
2450
# We won't be building this node
2451
del self._annotate_children[node]
2452
# Generally we will want to read the records in reverse order, because
2453
# we find the parent nodes after the children
2457
def _annotate_records(self, records):
2458
"""Build the annotations for the listed records."""
2459
# We iterate in the order read, rather than a strict order requested
2460
# However, process what we can, and put off to the side things that
2461
# still need parents, cleaning them up when those parents are
2463
for (rev_id, record,
2464
digest) in self._knit._read_records_iter(records):
2465
if rev_id in self._annotated_lines:
2467
parent_ids = self._revision_id_graph[rev_id]
2468
parent_ids = [p for p in parent_ids if p not in self._ghosts]
2469
details = self._all_build_details[rev_id]
2470
(index_memo, compression_parent, parents,
2471
record_details) = details
2472
nodes_to_annotate = []
2473
# TODO: Remove the punning between compression parents, and
2474
# parent_ids, we should be able to do this without assuming
2476
if len(parent_ids) == 0:
2477
# There are no parents for this node, so just add it
2478
# TODO: This probably needs to be decoupled
2479
fulltext_content, delta = self._knit._factory.parse_record(
2480
rev_id, record, record_details, None)
2481
fulltext = self._add_fulltext_content(rev_id, fulltext_content)
2482
nodes_to_annotate.extend(self._add_annotation(rev_id, fulltext,
2483
parent_ids, left_matching_blocks=None))
2485
child = (rev_id, parent_ids, record)
2486
# Check if all the parents are present
2487
self._check_parents(child, nodes_to_annotate)
2488
while nodes_to_annotate:
2489
# Should we use a queue here instead of a stack?
2490
(rev_id, parent_ids, record) = nodes_to_annotate.pop()
2491
(index_memo, compression_parent, parents,
2492
record_details) = self._all_build_details[rev_id]
2493
if compression_parent is not None:
2494
comp_children = self._compression_children[compression_parent]
2495
if rev_id not in comp_children:
2496
raise AssertionError("%r not in compression children %r"
2497
% (rev_id, comp_children))
2498
# If there is only 1 child, it is safe to reuse this
2500
reuse_content = (len(comp_children) == 1
2501
and compression_parent not in
2502
self._nodes_to_keep_annotations)
2504
# Remove it from the cache since it will be changing
2505
parent_fulltext_content = self._fulltext_contents.pop(compression_parent)
2506
# Make sure to copy the fulltext since it might be
2508
parent_fulltext = list(parent_fulltext_content.text())
2510
parent_fulltext_content = self._fulltext_contents[compression_parent]
2511
parent_fulltext = parent_fulltext_content.text()
2512
comp_children.remove(rev_id)
2513
fulltext_content, delta = self._knit._factory.parse_record(
2514
rev_id, record, record_details,
2515
parent_fulltext_content,
2516
copy_base_content=(not reuse_content))
2517
fulltext = self._add_fulltext_content(rev_id,
2519
blocks = KnitContent.get_line_delta_blocks(delta,
2520
parent_fulltext, fulltext)
2522
fulltext_content = self._knit._factory.parse_fulltext(
2524
fulltext = self._add_fulltext_content(rev_id,
2527
nodes_to_annotate.extend(
2528
self._add_annotation(rev_id, fulltext, parent_ids,
2529
left_matching_blocks=blocks))
2531
def _get_heads_provider(self):
2532
"""Create a heads provider for resolving ancestry issues."""
2533
if self._heads_provider is not None:
2534
return self._heads_provider
2535
parent_provider = _mod_graph.DictParentsProvider(
2536
self._revision_id_graph)
2537
graph_obj = _mod_graph.Graph(parent_provider)
2538
head_cache = _mod_graph.FrozenHeadsCache(graph_obj)
2539
self._heads_provider = head_cache
2542
def annotate(self, key):
2543
"""Return the annotated fulltext at the given key.
2545
:param key: The key to annotate.
2547
records = self._get_build_graph(key)
2548
if key in self._ghosts:
2549
raise errors.RevisionNotPresent(key, self._knit)
2550
self._annotate_records(records)
2551
return self._annotated_lines[key]
2555
from bzrlib._knit_load_data_c import _load_data_c as _load_data
2557
from bzrlib._knit_load_data_py import _load_data_py as _load_data
added
removed
bzrlib/knit.py
