572
529
def __init__(self, backing_vf):
573
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
530
"""Create a RecordingVersionedFileDsecorator decorating backing_vf.
575
532
:param backing_vf: The versioned file to answer all methods.
577
534
self._backing_vf = backing_vf
580
def add_lines(self, key, parents, lines, parent_texts=None,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
583
self.calls.append(("add_lines", key, parents, lines, parent_texts,
584
left_matching_blocks, nostore_sha, random_id, check_content))
585
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
586
left_matching_blocks, nostore_sha, random_id, check_content)
589
self._backing_vf.check()
591
def get_parent_map(self, keys):
592
self.calls.append(("get_parent_map", copy(keys)))
593
return self._backing_vf.get_parent_map(keys)
595
537
def get_record_stream(self, keys, sort_order, include_delta_closure):
596
self.calls.append(("get_record_stream", list(keys), sort_order,
538
self.calls.append(("get_record_stream", keys, sort_order,
597
539
include_delta_closure))
598
540
return self._backing_vf.get_record_stream(keys, sort_order,
599
541
include_delta_closure)
601
def get_sha1s(self, keys):
602
self.calls.append(("get_sha1s", copy(keys)))
603
return self._backing_vf.get_sha1s(keys)
605
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
606
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
607
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
610
self.calls.append(("keys",))
611
return self._backing_vf.keys()
614
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
615
"""A VF that records calls, and returns keys in specific order.
617
:ivar calls: A list of the calls made; can be reset at any time by
621
def __init__(self, backing_vf, key_priority):
622
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
624
:param backing_vf: The versioned file to answer all methods.
625
:param key_priority: A dictionary defining what order keys should be
626
returned from an 'unordered' get_record_stream request.
627
Keys with lower priority are returned first, keys not present in
628
the map get an implicit priority of 0, and are returned in
629
lexicographical order.
631
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
632
self._key_priority = key_priority
634
def get_record_stream(self, keys, sort_order, include_delta_closure):
635
self.calls.append(("get_record_stream", list(keys), sort_order,
636
include_delta_closure))
637
if sort_order == 'unordered':
639
return (self._key_priority.get(key, 0), key)
640
# Use a defined order by asking for the keys one-by-one from the
642
for key in sorted(keys, key=sort_key):
643
for record in self._backing_vf.get_record_stream([key],
644
'unordered', include_delta_closure):
647
for record in self._backing_vf.get_record_stream(keys, sort_order,
648
include_delta_closure):
652
544
class KeyMapper(object):
653
545
"""KeyMappers map between keys and underlying partitioned storage."""
1478
1283
elif state == 'conflicted-b':
1479
1284
ch_b = ch_a = True
1480
1285
lines_b.append(line)
1481
elif state == 'killed-both':
1482
# This counts as a change, even though there is no associated
1486
1287
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1288
'killed-base', 'killed-both'):
1488
1289
raise AssertionError(state)
1489
1290
for struct in outstanding_struct():
1492
def base_from_plan(self):
1493
"""Construct a BASE file from the plan text."""
1495
for state, line in self.plan:
1496
if state in ('killed-a', 'killed-b', 'killed-both', 'unchanged'):
1497
# If unchanged, then this line is straight from base. If a or b
1498
# or both killed the line, then it *used* to be in base.
1499
base_lines.append(line)
1501
if state not in ('killed-base', 'irrelevant',
1502
'ghost-a', 'ghost-b',
1504
'conflicted-a', 'conflicted-b'):
1505
# killed-base, irrelevant means it doesn't apply
1506
# ghost-a/ghost-b are harder to say for sure, but they
1507
# aren't in the 'inc_c' which means they aren't in the
1508
# shared base of a & b. So we don't include them. And
1509
# obviously if the line is newly inserted, it isn't in base
1511
# If 'conflicted-a' or b, then it is new vs one base, but
1512
# old versus another base. However, if we make it present
1513
# in the base, it will be deleted from the target, and it
1514
# seems better to get a line doubled in the merge result,
1515
# rather than have it deleted entirely.
1516
# Example, each node is the 'text' at that point:
1524
# There was a criss-cross conflict merge. Both sides
1525
# include the other, but put themselves first.
1526
# Weave marks this as a 'clean' merge, picking OTHER over
1527
# THIS. (Though the details depend on order inserted into
1529
# LCA generates a plan:
1530
# [('unchanged', M),
1531
# ('conflicted-b', b),
1533
# ('conflicted-a', b),
1535
# If you mark 'conflicted-*' as part of BASE, then a 3-way
1536
# merge tool will cleanly generate "MaN" (as BASE vs THIS
1537
# removes one 'b', and BASE vs OTHER removes the other)
1538
# If you include neither, 3-way creates a clean "MbabN" as
1539
# THIS adds one 'b', and OTHER does too.
1540
# It seems that having the line 2 times is better than
1541
# having it omitted. (Easier to manually delete than notice
1542
# it needs to be added.)
1543
raise AssertionError('Unknown state: %s' % (state,))
1547
1294
class WeaveMerge(PlanWeaveMerge):
1548
1295
"""Weave merge that takes a VersionedFile and two versions as its input."""
1550
def __init__(self, versionedfile, ver_a, ver_b,
1297
def __init__(self, versionedfile, ver_a, ver_b,
1551
1298
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1552
1299
plan = versionedfile.plan_merge(ver_a, ver_b)
1553
1300
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1556
class VirtualVersionedFiles(VersionedFiles):
1557
"""Dummy implementation for VersionedFiles that uses other functions for
1558
obtaining fulltexts and parent maps.
1560
This is always on the bottom of the stack and uses string keys
1561
(rather than tuples) internally.
1564
def __init__(self, get_parent_map, get_lines):
1565
"""Create a VirtualVersionedFiles.
1567
:param get_parent_map: Same signature as Repository.get_parent_map.
1568
:param get_lines: Should return lines for specified key or None if
1571
super(VirtualVersionedFiles, self).__init__()
1572
self._get_parent_map = get_parent_map
1573
self._get_lines = get_lines
1575
def check(self, progressbar=None):
1576
"""See VersionedFiles.check.
1578
:note: Always returns True for VirtualVersionedFiles.
1582
def add_mpdiffs(self, records):
1583
"""See VersionedFiles.mpdiffs.
1585
:note: Not implemented for VirtualVersionedFiles.
1587
raise NotImplementedError(self.add_mpdiffs)
1589
def get_parent_map(self, keys):
1590
"""See VersionedFiles.get_parent_map."""
1591
return dict([((k,), tuple([(p,) for p in v]))
1592
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1594
def get_sha1s(self, keys):
1595
"""See VersionedFiles.get_sha1s."""
1598
lines = self._get_lines(k)
1599
if lines is not None:
1600
if not isinstance(lines, list):
1601
raise AssertionError
1602
ret[(k,)] = osutils.sha_strings(lines)
1605
def get_record_stream(self, keys, ordering, include_delta_closure):
1606
"""See VersionedFiles.get_record_stream."""
1607
for (k,) in list(keys):
1608
lines = self._get_lines(k)
1609
if lines is not None:
1610
if not isinstance(lines, list):
1611
raise AssertionError
1612
yield ChunkedContentFactory((k,), None,
1613
sha1=osutils.sha_strings(lines),
1616
yield AbsentContentFactory((k,))
1618
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1619
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1620
for i, (key,) in enumerate(keys):
1622
pb.update("Finding changed lines", i, len(keys))
1623
for l in self._get_lines(key):
1627
def network_bytes_to_kind_and_offset(network_bytes):
1628
"""Strip of a record kind from the front of network_bytes.
1630
:param network_bytes: The bytes of a record.
1631
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1635
return storage_kind, line_end + 1
1638
class NetworkRecordStream(object):
1639
"""A record_stream which reconstitures a serialised stream."""
1641
def __init__(self, bytes_iterator):
1642
"""Create a NetworkRecordStream.
1644
:param bytes_iterator: An iterator of bytes. Each item in this
1645
iterator should have been obtained from a record_streams'
1646
record.get_bytes_as(record.storage_kind) call.
1648
self._bytes_iterator = bytes_iterator
1649
self._kind_factory = {
1650
'fulltext': fulltext_network_to_record,
1651
'groupcompress-block': groupcompress.network_block_to_records,
1652
'knit-ft-gz': knit.knit_network_to_record,
1653
'knit-delta-gz': knit.knit_network_to_record,
1654
'knit-annotated-ft-gz': knit.knit_network_to_record,
1655
'knit-annotated-delta-gz': knit.knit_network_to_record,
1656
'knit-delta-closure': knit.knit_delta_closure_to_records,
1662
:return: An iterator as per VersionedFiles.get_record_stream().
1664
for bytes in self._bytes_iterator:
1665
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1666
for record in self._kind_factory[storage_kind](
1667
storage_kind, bytes, line_end):
1671
def fulltext_network_to_record(kind, bytes, line_end):
1672
"""Convert a network fulltext record to record."""
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1675
key, parents = bencode.bdecode_as_tuple(record_meta)
1676
if parents == 'nil':
1678
fulltext = bytes[line_end+4+meta_len:]
1679
return [FulltextContentFactory(key, parents, None, fulltext)]
1682
def _length_prefix(bytes):
1683
return struct.pack('!L', len(bytes))
1686
def record_to_fulltext_bytes(record):
1687
if record.parents is None:
1690
parents = record.parents
1691
record_meta = bencode.bencode((record.key, parents))
1692
record_content = record.get_bytes_as('fulltext')
1693
return "fulltext\n%s%s%s" % (
1694
_length_prefix(record_meta), record_meta, record_content)
1697
def sort_groupcompress(parent_map):
1698
"""Sort and group the keys in parent_map into groupcompress order.
1700
groupcompress is defined (currently) as reverse-topological order, grouped
1703
:return: A sorted-list of keys
1705
# gc-optimal ordering is approximately reverse topological,
1706
# properly grouped by file-id.
1708
for item in parent_map.iteritems():
1710
if isinstance(key, str) or len(key) == 1:
1715
per_prefix_map[prefix].append(item)
1717
per_prefix_map[prefix] = [item]
1720
for prefix in sorted(per_prefix_map):
1721
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
added
removed
bzrlib/versionedfile.py
