/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/bzr/index.py

  • Committer: Jelmer Vernooij
  • Date: 2019-05-29 03:22:34 UTC
  • mfrom: (7303 work)
  • mto: This revision was merged to the branch mainline in revision 7306.
  • Revision ID: jelmer@jelmer.uk-20190529032234-mt3fuws8gq03tapi
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/bzr/index.py
248
248
        :param value: The value associate with this key. Must not contain
249
249
            newlines or null characters.
250
250
        :return: (node_refs, absent_references)
251
 
        
 
251
 
252
252
            * node_refs: basically a packed form of 'references' where all
253
253
              iterables are tuples
254
254
            * absent_references: reference keys that are not in self._nodes.
306
306
        This is a no-op, but we need the api to conform to a generic 'Index'
307
307
        abstraction.
308
308
        """
309
 
        
 
309
 
310
310
    def finish(self):
311
311
        """Finish the index.
312
312
 
313
 
        :returns: cBytesIO holding the full context of the index as it 
 
313
        :returns: cBytesIO holding the full context of the index as it
314
314
        should be written to disk.
315
315
        """
316
316
        lines = [_SIGNATURE]
370
370
                            non_ref_bytes += len(ref_list) - 1
371
371
            # how many digits are needed to represent the total byte count?
372
372
            digits = 1
373
 
            possible_total_bytes = non_ref_bytes + total_references*digits
 
373
            possible_total_bytes = non_ref_bytes + total_references * digits
374
374
            while 10 ** digits < possible_total_bytes:
375
375
                digits += 1
376
 
                possible_total_bytes = non_ref_bytes + total_references*digits
377
 
            expected_bytes = possible_total_bytes + 1 # terminating newline
 
376
                possible_total_bytes = non_ref_bytes + total_references * digits
 
377
            expected_bytes = possible_total_bytes + 1  # terminating newline
378
378
            # resolve key addresses.
379
379
            key_addresses = {}
380
380
            for key, non_ref_bytes, total_references in key_offset_info:
381
 
                key_addresses[key] = non_ref_bytes + total_references*digits
 
381
                key_addresses[key] = non_ref_bytes + total_references * digits
382
382
            # serialise
383
383
            format_string = b'%%0%dd' % digits
384
384
        for key, (absent, references, value) in nodes:
386
386
            for ref_list in references:
387
387
                ref_addresses = []
388
388
                for reference in ref_list:
389
 
                    ref_addresses.append(format_string % key_addresses[reference])
 
389
                    ref_addresses.append(format_string %
 
390
                                         key_addresses[reference])
390
391
                flattened_references.append(b'\r'.join(ref_addresses))
391
392
            string_key = b'\x00'.join(key)
392
393
            lines.append(b"%s\x00%s\x00%s\x00%s\n" % (string_key, absent,
393
 
                b'\t'.join(flattened_references), value))
 
394
                                                      b'\t'.join(flattened_references), value))
394
395
        lines.append(b'\n')
395
396
        result = BytesIO(b''.join(lines))
396
397
        if expected_bytes and len(result.getvalue()) != expected_bytes:
397
398
            raise errors.BzrError('Failed index creation. Internal error:'
398
 
                ' mismatched output length and expected length: %d %d' %
399
 
                (len(result.getvalue()), expected_bytes))
 
399
                                  ' mismatched output length and expected length: %d %d' %
 
400
                                  (len(result.getvalue()), expected_bytes))
400
401
        return result
401
402
 
402
403
    def set_optimize(self, for_size=None, combine_backing_indices=None):
504
505
    def __lt__(self, other):
505
506
        # We don't really care about the order, just that there is an order.
506
507
        if (not isinstance(other, GraphIndex) and
507
 
            not isinstance(other, InMemoryGraphIndex)):
 
508
                not isinstance(other, InMemoryGraphIndex)):
508
509
            raise TypeError(other)
509
510
        return hash(self) < hash(other)
510
511
 
513
514
 
514
515
    def __repr__(self):
515
516
        return "%s(%r)" % (self.__class__.__name__,
516
 
            self._transport.abspath(self._name))
 
517
                           self._transport.abspath(self._name))
517
518
 
518
519
    def _buffer_all(self, stream=None):
519
520
        """Buffer all the index data.
525
526
            return
526
527
        if 'index' in debug.debug_flags:
527
528
            trace.mutter('Reading entire index %s',
528
 
                          self._transport.abspath(self._name))
 
529
                         self._transport.abspath(self._name))
529
530
        if stream is None:
530
531
            stream = self._transport.get(self._name)
531
532
            if self._base_offset != 0:
576
577
        self._buffer_all()
577
578
        if ref_list_num + 1 > self.node_ref_lists:
578
579
            raise ValueError('No ref list %d, index has %d ref lists'
579
 
                % (ref_list_num, self.node_ref_lists))
 
580
                             % (ref_list_num, self.node_ref_lists))
580
581
        refs = set()
581
582
        nodes = self._nodes
582
583
        for key, (value, ref_lists) in viewitems(nodes):
613
614
        """
614
615
        if 'evil' in debug.debug_flags:
615
616
            trace.mutter_callsite(3,
616
 
                "iter_all_entries scales with size of history.")
 
617
                                  "iter_all_entries scales with size of history.")
617
618
        if self._nodes is None:
618
619
            self._buffer_all()
619
620
        if self.node_ref_lists:
661
662
        """
662
663
        node_refs = []
663
664
        for ref_list in references:
664
 
            node_refs.append(tuple([self._keys_by_offset[ref][0] for ref in ref_list]))
 
665
            node_refs.append(
 
666
                tuple([self._keys_by_offset[ref][0] for ref in ref_list]))
665
667
        return tuple(node_refs)
666
668
 
667
669
    @staticmethod
866
868
            index = self._parsed_byte_index(location)
867
869
            if (len(self._parsed_byte_map) and
868
870
                self._parsed_byte_map[index][0] <= location and
869
 
                self._parsed_byte_map[index][1] > location):
 
871
                    self._parsed_byte_map[index][1] > location):
870
872
                # the byte region has been parsed, so no read is needed.
871
873
                continue
872
874
            length = 800
884
886
            # _read_and_parse triggered a _buffer_all because we requested the
885
887
            # whole data range
886
888
            for location, key in location_keys:
887
 
                if key not in self._nodes: # not present
 
889
                if key not in self._nodes:  # not present
888
890
                    result.append(((location, key), False))
889
891
                elif self.node_ref_lists:
890
892
                    value, refs = self._nodes[key]
891
893
                    result.append(((location, key),
892
 
                        (self, key, value, refs)))
 
894
                                   (self, key, value, refs)))
893
895
                else:
894
896
                    result.append(((location, key),
895
 
                        (self, key, self._nodes[key])))
 
897
                                   (self, key, self._nodes[key])))
896
898
            return result
897
899
        # generate results:
898
900
        #  - figure out <, >, missing, present
917
919
                        pending_references.append((location, key))
918
920
                        continue
919
921
                    result.append(((location, key), (self, key,
920
 
                        value, self._resolve_references(refs))))
 
922
                                                     value, self._resolve_references(refs))))
921
923
                else:
922
924
                    result.append(((location, key),
923
 
                        (self, key, self._bisect_nodes[key])))
 
925
                                   (self, key, self._bisect_nodes[key])))
924
926
                continue
925
927
            else:
926
928
                # has the region the key should be in, been parsed?
963
965
            # answer key references we had to look-up-late.
964
966
            value, refs = self._bisect_nodes[key]
965
967
            result.append(((location, key), (self, key,
966
 
                value, self._resolve_references(refs))))
 
968
                                             value, self._resolve_references(refs))))
967
969
        return result
968
970
 
969
971
    def _parse_header_from_bytes(self, bytes):
1000
1002
            raise BadIndexOptions(self)
1001
1003
        # calculate the bytes we have processed
1002
1004
        header_end = (len(signature) + len(lines[0]) + len(lines[1]) +
1003
 
            len(lines[2]) + 3)
 
1005
                      len(lines[2]) + 3)
1004
1006
        self._parsed_bytes(0, (), header_end, ())
1005
1007
        # setup parsing state
1006
1008
        self._expected_elements = 3 + self._key_length
1126
1128
        trimmed_data = data[trim_start:trim_end]
1127
1129
        if not (trimmed_data):
1128
1130
            raise AssertionError('read unneeded data [%d:%d] from [%d:%d]'
1129
 
                % (trim_start, trim_end, offset, offset + len(data)))
 
1131
                                 % (trim_start, trim_end, offset, offset + len(data)))
1130
1132
        if trim_start:
1131
1133
            offset += trim_start
1132
1134
        # print "parsing", repr(trimmed_data)
1138
1140
        for key, value in nodes:
1139
1141
            self._bisect_nodes[key] = value
1140
1142
        self._parsed_bytes(offset, first_key,
1141
 
            offset + len(trimmed_data), last_key)
 
1143
                           offset + len(trimmed_data), last_key)
1142
1144
        return offset + len(trimmed_data), last_segment
1143
1145
 
1144
1146
    def _parse_lines(self, lines, pos):
1159
1161
                raise BadIndexData(self)
1160
1162
            # keys are tuples. Each element is a string that may occur many
1161
1163
            # times, so we intern them to save space. AB, RC, 200807
1162
 
            key = tuple([bytesintern(element) for element in elements[:self._key_length]])
 
1164
            key = tuple([bytesintern(element)
 
1165
                         for element in elements[:self._key_length]])
1163
1166
            if first_key is None:
1164
1167
                first_key = key
1165
1168
            absent, references, value = elements[-3:]
1170
1173
                    ]))
1171
1174
            ref_lists = tuple(ref_lists)
1172
1175
            self._keys_by_offset[pos] = (key, absent, ref_lists, value)
1173
 
            pos += len(line) + 1 # +1 for the \n
 
1176
            pos += len(line) + 1  # +1 for the \n
1174
1177
            if absent:
1175
1178
                continue
1176
1179
            if self.node_ref_lists:
1205
1208
        # combine two regions
1206
1209
        if (index + 1 < len(self._parsed_byte_map) and
1207
1210
            self._parsed_byte_map[index][1] == start and
1208
 
            self._parsed_byte_map[index + 1][0] == end):
 
1211
                self._parsed_byte_map[index + 1][0] == end):
1209
1212
            # combine two regions
1210
1213
            self._parsed_byte_map[index] = (self._parsed_byte_map[index][0],
1211
 
                self._parsed_byte_map[index + 1][1])
 
1214
                                            self._parsed_byte_map[index + 1][1])
1212
1215
            self._parsed_key_map[index] = (self._parsed_key_map[index][0],
1213
 
                self._parsed_key_map[index + 1][1])
 
1216
                                           self._parsed_key_map[index + 1][1])
1214
1217
            del self._parsed_byte_map[index + 1]
1215
1218
            del self._parsed_key_map[index + 1]
1216
1219
        elif self._parsed_byte_map[index][1] == start:
1220
1223
            self._parsed_key_map[index] = (
1221
1224
                self._parsed_key_map[index][0], end_key)
1222
1225
        elif (index + 1 < len(self._parsed_byte_map) and
1223
 
            self._parsed_byte_map[index + 1][0] == end):
 
1226
              self._parsed_byte_map[index + 1][0] == end):
1224
1227
            # extend the higher entry
1225
1228
            self._parsed_byte_map[index + 1] = (
1226
1229
                start, self._parsed_byte_map[index + 1][1])
1247
1250
        base_offset = self._base_offset
1248
1251
        if base_offset != 0:
1249
1252
            # Rewrite the ranges for the offset
1250
 
            readv_ranges = [(start+base_offset, size)
 
1253
            readv_ranges = [(start + base_offset, size)
1251
1254
                            for start, size in readv_ranges]
1252
1255
        readv_data = self._transport.readv(self._name, readv_ranges, True,
1253
 
            self._size + self._base_offset)
 
1256
                                           self._size + self._base_offset)
1254
1257
        # parse
1255
1258
        for offset, data in readv_data:
1256
1259
            offset -= base_offset
1296
1299
    performance significantly. For example, if one index is on local disk and a
1297
1300
    second on a remote server, the local disk index should be before the other
1298
1301
    in the index list.
1299
 
    
 
1302
 
1300
1303
    Also, queries tend to need results from the same indices as previous
1301
1304
    queries.  So the indices will be reordered after every query to put the
1302
1305
    indices that had the result(s) of that query first (while otherwise
1323
1326
 
1324
1327
    def __repr__(self):
1325
1328
        return "%s(%s)" % (
1326
 
                self.__class__.__name__,
1327
 
                ', '.join(map(repr, self._indices)))
 
1329
            self.__class__.__name__,
 
1330
            ', '.join(map(repr, self._indices)))
1328
1331
 
1329
1332
    def clear_cache(self):
1330
1333
        """See GraphIndex.clear_cache()"""
1336
1339
        search_keys = set(keys)
1337
1340
        if _mod_revision.NULL_REVISION in search_keys:
1338
1341
            search_keys.discard(_mod_revision.NULL_REVISION)
1339
 
            found_parents = {_mod_revision.NULL_REVISION:[]}
 
1342
            found_parents = {_mod_revision.NULL_REVISION: []}
1340
1343
        else:
1341
1344
            found_parents = {}
1342
1345
        for index, key, value, refs in self.iter_entries(search_keys):
1478
1481
 
1479
1482
    def _move_to_front_by_index(self, hit_indices):
1480
1483
        """Core logic for _move_to_front.
1481
 
        
 
1484
 
1482
1485
        Returns a list of names corresponding to the hit_indices param.
1483
1486
        """
1484
1487
        indices_info = zip(self._index_names, self._indices)
1498
1501
                if len(new_hit_indices) == len(hit_indices):
1499
1502
                    # We've found all of the hit entries, everything else is
1500
1503
                    # unhit
1501
 
                    unhit_names.extend(self._index_names[offset+1:])
1502
 
                    unhit_indices.extend(self._indices[offset+1:])
 
1504
                    unhit_names.extend(self._index_names[offset + 1:])
 
1505
                    unhit_indices.extend(self._indices[offset + 1:])
1503
1506
                    break
1504
1507
            else:
1505
1508
                unhit_names.append(name)
1567
1570
                    #       CombinedGraphIndex does not know what the ref lists
1568
1571
                    #       mean.
1569
1572
                    search_keys = index._find_ancestors(search_keys,
1570
 
                        ref_list_num, parent_map, index_missing_keys)
 
1573
                                                        ref_list_num, parent_map, index_missing_keys)
1571
1574
                    # print '    \t  \t%2d\t%4d\t%5d\t%5d' % (
1572
1575
                    #     sub_generation, len(search_keys),
1573
1576
                    #     len(parent_map), len(index_missing_keys))
1613
1616
        """
1614
1617
        if self._reload_func is None:
1615
1618
            return False
1616
 
        trace.mutter('Trying to reload after getting exception: %s', str(error))
 
1619
        trace.mutter(
 
1620
            'Trying to reload after getting exception: %s', str(error))
1617
1621
        if not self._reload_func():
1618
1622
            # We tried to reload, but nothing changed, so we fail anyway
1619
1623
            trace.mutter('_reload_func indicated nothing has changed.'
1667
1671
        """
1668
1672
        if 'evil' in debug.debug_flags:
1669
1673
            trace.mutter_callsite(3,
1670
 
                "iter_all_entries scales with size of history.")
 
1674
                                  "iter_all_entries scales with size of history.")
1671
1675
        if self.reference_lists:
1672
1676
            for key, (absent, references, value) in viewitems(self._nodes):
1673
1677
                if not absent:
1748
1752
    def __lt__(self, other):
1749
1753
        # We don't really care about the order, just that there is an order.
1750
1754
        if (not isinstance(other, GraphIndex) and
1751
 
            not isinstance(other, InMemoryGraphIndex)):
 
1755
                not isinstance(other, InMemoryGraphIndex)):
1752
1756
            raise TypeError(other)
1753
1757
        return hash(self) < hash(other)
1754
1758
 
1764
1768
    """
1765
1769
 
1766
1770
    def __init__(self, adapted, prefix, missing_key_length,
1767
 
        add_nodes_callback=None):
 
1771
                 add_nodes_callback=None):
1768
1772
        """Construct an adapter against adapted with prefix."""
1769
1773
        self.adapted = adapted
1770
 
        self.prefix_key = prefix + (None,)*missing_key_length
 
1774
        self.prefix_key = prefix + (None,) * missing_key_length
1771
1775
        self.prefix = prefix
1772
1776
        self.prefix_len = len(prefix)
1773
1777
        self.add_nodes_callback = add_nodes_callback
1786
1790
            for (key, value, node_refs) in nodes:
1787
1791
                adjusted_references = (
1788
1792
                    tuple(tuple(self.prefix + ref_node for ref_node in ref_list)
1789
 
                        for ref_list in node_refs))
 
1793
                          for ref_list in node_refs))
1790
1794
                translated_nodes.append((self.prefix + key, value,
1791
 
                    adjusted_references))
 
1795
                                         adjusted_references))
1792
1796
        except ValueError:
1793
1797
            # XXX: TODO add an explicit interface for getting the reference list
1794
1798
            # status, to handle this bit of user-friendliness in the API more
1822
1826
                        raise BadIndexData(self)
1823
1827
            yield node[0], node[1][self.prefix_len:], node[2], (
1824
1828
                tuple(tuple(ref_node[self.prefix_len:] for ref_node in ref_list)
1825
 
                for ref_list in node[3]))
 
1829
                      for ref_list in node[3]))
1826
1830
 
1827
1831
    def iter_all_entries(self):
1828
1832
        """Iterate over all keys within the index