/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/knit.py

  • Committer: John Arbash Meinel
  • Date: 2009-08-17 15:38:52 UTC
  • mfrom: (4615.1.1 2.0b1-merge-sort)
  • mto: This revision was merged to the branch mainline in revision 4629.
  • Revision ID: john@arbash-meinel.com-20090817153852-9127xf3aabmcoegt
Bring in the changes to merge_sort.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/knit.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
69
69
    lru_cache,
70
70
    pack,
71
71
    progress,
72
 
    static_tuple,
73
72
    trace,
74
73
    tsort,
75
74
    tuned_gzip,
76
 
    ui,
77
75
    )
78
76
""")
79
77
from bzrlib import (
1194
1192
 
1195
1193
    def get_known_graph_ancestry(self, keys):
1196
1194
        """Get a KnownGraph instance with the ancestry of keys."""
1197
 
        parent_map, missing_keys = self._index.find_ancestry(keys)
1198
 
        for fallback in self._fallback_vfs:
1199
 
            if not missing_keys:
1200
 
                break
1201
 
            (f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1202
 
                                                missing_keys)
1203
 
            parent_map.update(f_parent_map)
1204
 
            missing_keys = f_missing_keys
 
1195
        parent_map, missing_keys = self._index._graph_index.find_ancestry(keys,
 
1196
                                                                          0)
1205
1197
        kg = _mod_graph.KnownGraph(parent_map)
1206
1198
        return kg
1207
1199
 
1520
1512
                if source is parent_maps[0]:
1521
1513
                    # this KnitVersionedFiles
1522
1514
                    records = [(key, positions[key][1]) for key in keys]
1523
 
                    for key, raw_data in self._read_records_iter_unchecked(records):
 
1515
                    for key, raw_data, sha1 in self._read_records_iter_raw(records):
1524
1516
                        (record_details, index_memo, _) = positions[key]
1525
1517
                        yield KnitContentFactory(key, global_map[key],
1526
 
                            record_details, None, raw_data, self._factory.annotated, None)
 
1518
                            record_details, sha1, raw_data, self._factory.annotated, None)
1527
1519
                else:
1528
1520
                    vf = self._fallback_vfs[parent_maps.index(source) - 1]
1529
1521
                    for record in vf.get_record_stream(keys, ordering,
1598
1590
        # key = basis_parent, value = index entry to add
1599
1591
        buffered_index_entries = {}
1600
1592
        for record in stream:
1601
 
            kind = record.storage_kind
1602
 
            if kind.startswith('knit-') and kind.endswith('-gz'):
1603
 
                # Check that the ID in the header of the raw knit bytes matches
1604
 
                # the record metadata.
1605
 
                raw_data = record._raw_record
1606
 
                df, rec = self._parse_record_header(record.key, raw_data)
1607
 
                df.close()
1608
1593
            buffered = False
1609
1594
            parents = record.parents
1610
1595
            if record.storage_kind in delta_types:
1712
1697
            # There were index entries buffered at the end of the stream,
1713
1698
            # So these need to be added (if the index supports holding such
1714
1699
            # entries for later insertion)
1715
 
            all_entries = []
1716
1700
            for key in buffered_index_entries:
1717
1701
                index_entries = buffered_index_entries[key]
1718
 
                all_entries.extend(index_entries)
1719
 
            self._index.add_records(
1720
 
                all_entries, missing_compression_parents=True)
 
1702
                self._index.add_records(index_entries,
 
1703
                    missing_compression_parents=True)
1721
1704
 
1722
1705
    def get_missing_compression_parent_keys(self):
1723
1706
        """Return an iterable of keys of missing compression parents.
1756
1739
        :return: An iterator over (line, key).
1757
1740
        """
1758
1741
        if pb is None:
1759
 
            pb = ui.ui_factory.nested_progress_bar()
 
1742
            pb = progress.DummyProgress()
1760
1743
        keys = set(keys)
1761
1744
        total = len(keys)
1762
1745
        done = False
2369
2352
    FLAGS is a comma separated list of flags about the record. Values include
2370
2353
        no-eol, line-delta, fulltext.
2371
2354
    BYTE_OFFSET is the ascii representation of the byte offset in the data file
2372
 
        that the compressed data starts at.
 
2355
        that the the compressed data starts at.
2373
2356
    LENGTH is the ascii representation of the length of the data file.
2374
2357
    PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
2375
2358
        REVISION_ID.
2584
2567
        except KeyError:
2585
2568
            raise RevisionNotPresent(key, self)
2586
2569
 
2587
 
    def find_ancestry(self, keys):
2588
 
        """See CombinedGraphIndex.find_ancestry()"""
2589
 
        prefixes = set(key[:-1] for key in keys)
2590
 
        self._load_prefixes(prefixes)
2591
 
        result = {}
2592
 
        parent_map = {}
2593
 
        missing_keys = set()
2594
 
        pending_keys = list(keys)
2595
 
        # This assumes that keys will not reference parents in a different
2596
 
        # prefix, which is accurate so far.
2597
 
        while pending_keys:
2598
 
            key = pending_keys.pop()
2599
 
            if key in parent_map:
2600
 
                continue
2601
 
            prefix = key[:-1]
2602
 
            try:
2603
 
                suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
2604
 
            except KeyError:
2605
 
                missing_keys.add(key)
2606
 
            else:
2607
 
                parent_keys = tuple([prefix + (suffix,)
2608
 
                                     for suffix in suffix_parents])
2609
 
                parent_map[key] = parent_keys
2610
 
                pending_keys.extend([p for p in parent_keys
2611
 
                                        if p not in parent_map])
2612
 
        return parent_map, missing_keys
2613
 
 
2614
2570
    def get_parent_map(self, keys):
2615
2571
        """Get a map of the parents of keys.
2616
2572
 
2788
2744
 
2789
2745
class _KeyRefs(object):
2790
2746
 
2791
 
    def __init__(self, track_new_keys=False):
 
2747
    def __init__(self):
2792
2748
        # dict mapping 'key' to 'set of keys referring to that key'
2793
2749
        self.refs = {}
2794
 
        if track_new_keys:
2795
 
            # set remembering all new keys
2796
 
            self.new_keys = set()
2797
 
        else:
2798
 
            self.new_keys = None
2799
 
 
2800
 
    def clear(self):
2801
 
        if self.refs:
2802
 
            self.refs.clear()
2803
 
        if self.new_keys:
2804
 
            self.new_keys.clear()
2805
2750
 
2806
2751
    def add_references(self, key, refs):
2807
2752
        # Record the new references
2814
2759
        # Discard references satisfied by the new key
2815
2760
        self.add_key(key)
2816
2761
 
2817
 
    def get_new_keys(self):
2818
 
        return self.new_keys
2819
 
    
2820
2762
    def get_unsatisfied_refs(self):
2821
2763
        return self.refs.iterkeys()
2822
2764
 
2823
 
    def _satisfy_refs_for_key(self, key):
 
2765
    def add_key(self, key):
2824
2766
        try:
2825
2767
            del self.refs[key]
2826
2768
        except KeyError:
2827
2769
            # No keys depended on this key.  That's ok.
2828
2770
            pass
2829
2771
 
2830
 
    def add_key(self, key):
2831
 
        # satisfy refs for key, and remember that we've seen this key.
2832
 
        self._satisfy_refs_for_key(key)
2833
 
        if self.new_keys is not None:
2834
 
            self.new_keys.add(key)
2835
 
 
2836
 
    def satisfy_refs_for_keys(self, keys):
 
2772
    def add_keys(self, keys):
2837
2773
        for key in keys:
2838
 
            self._satisfy_refs_for_key(key)
 
2774
            self.add_key(key)
2839
2775
 
2840
2776
    def get_referrers(self):
2841
2777
        result = set()
2946
2882
        if not random_id:
2947
2883
            present_nodes = self._get_entries(keys)
2948
2884
            for (index, key, value, node_refs) in present_nodes:
2949
 
                parents = node_refs[:1]
2950
 
                # Sometimes these are passed as a list rather than a tuple
2951
 
                passed = static_tuple.as_tuples(keys[key])
2952
 
                passed_parents = passed[1][:1]
2953
2885
                if (value[0] != keys[key][0][0] or
2954
 
                    parents != passed_parents):
2955
 
                    node_refs = static_tuple.as_tuples(node_refs)
 
2886
                    node_refs[:1] != keys[key][1][:1]):
2956
2887
                    raise KnitCorrupt(self, "inconsistent details in add_records"
2957
 
                        ": %s %s" % ((value, node_refs), passed))
 
2888
                        ": %s %s" % ((value, node_refs), keys[key]))
2958
2889
                del keys[key]
2959
2890
        result = []
2960
2891
        if self._parents:
3008
2939
        # If updating this, you should also update
3009
2940
        # groupcompress._GCGraphIndex.get_missing_parents
3010
2941
        # We may have false positives, so filter those out.
3011
 
        self._key_dependencies.satisfy_refs_for_keys(
 
2942
        self._key_dependencies.add_keys(
3012
2943
            self.get_parent_map(self._key_dependencies.get_unsatisfied_refs()))
3013
2944
        return frozenset(self._key_dependencies.get_unsatisfied_refs())
3014
2945
 
3125
3056
            options.append('no-eol')
3126
3057
        return options
3127
3058
 
3128
 
    def find_ancestry(self, keys):
3129
 
        """See CombinedGraphIndex.find_ancestry()"""
3130
 
        return self._graph_index.find_ancestry(keys, 0)
3131
 
 
3132
3059
    def get_parent_map(self, keys):
3133
3060
        """Get a map of the parents of keys.
3134
3061
 
3709
3636
 
3710
3637
try:
3711
3638
    from bzrlib._knit_load_data_pyx import _load_data_c as _load_data
3712
 
except ImportError, e:
3713
 
    osutils.failed_to_load_extension(e)
 
3639
except ImportError:
3714
3640
    from bzrlib._knit_load_data_py import _load_data_py as _load_data