/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
17
from bzrlib import (
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
18
    debug,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
19
    errors,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
20
    revision,
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
21
    symbol_versioning,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
22
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
23
    tsort,
24
    )
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
25
from bzrlib.deprecated_graph import (node_distances, select_farthest)
2490.2.1 by Aaron Bentley
Start work on GraphWalker
26
2490.2.25 by Aaron Bentley
Update from review
27
# DIAGRAM of terminology
28
#       A
29
#       /\
30
#      B  C
31
#      |  |\
32
#      D  E F
33
#      |\/| |
34
#      |/\|/
35
#      G  H
36
#
37
# In this diagram, relative to G and H:
38
# A, B, C, D, E are common ancestors.
39
# C, D and E are border ancestors, because each has a non-common descendant.
40
# D and E are least common ancestors because none of their descendants are
41
# common ancestors.
42
# C is not a least common ancestor because its descendant, E, is a common
43
# ancestor.
44
#
45
# The find_unique_lca algorithm will pick A in two steps:
46
# 1. find_lca('G', 'H') => ['D', 'E']
47
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
48
49
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
50
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
51
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
52
53
    def __init__(self, ancestry):
54
        self.ancestry = ancestry
55
56
    def __repr__(self):
57
        return 'DictParentsProvider(%r)' % self.ancestry
58
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
59
    def get_parent_map(self, keys):
60
        """See _StackedParentsProvider.get_parent_map"""
61
        ancestry = self.ancestry
62
        return dict((k, ancestry[k]) for k in keys if k in ancestry)
63
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
64
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
65
class _StackedParentsProvider(object):
66
67
    def __init__(self, parent_providers):
68
        self._parent_providers = parent_providers
69
2490.2.28 by Aaron Bentley
Fix handling of null revision
70
    def __repr__(self):
71
        return "_StackedParentsProvider(%r)" % self._parent_providers
72
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
73
    def get_parent_map(self, keys):
74
        """Get a mapping of keys => parents
75
76
        A dictionary is returned with an entry for each key present in this
77
        source. If this source doesn't have information about a key, it should
78
        not include an entry.
79
80
        [NULL_REVISION] is used as the parent of the first user-committed
81
        revision.  Its parent list is empty.
82
83
        :param keys: An iterable returning keys to check (eg revision_ids)
84
        :return: A dictionary mapping each key to its parents
85
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
86
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
87
        remaining = set(keys)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
88
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
89
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
90
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
91
            remaining.difference_update(new_found)
92
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
93
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
94
        return found
95
96
97
class CachingParentsProvider(object):
98
    """A parents provider which will cache the revision => parents in a dict.
99
100
    This is useful for providers that have an expensive lookup.
101
    """
102
103
    def __init__(self, parent_provider):
104
        self._real_provider = parent_provider
105
        # Theoretically we could use an LRUCache here
106
        self._cache = {}
107
108
    def __repr__(self):
109
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
110
111
    def get_parent_map(self, keys):
112
        """See _StackedParentsProvider.get_parent_map"""
113
        needed = set()
114
        # If the _real_provider doesn't have a key, we cache a value of None,
115
        # which we then later use to realize we cannot provide a value for that
116
        # key.
117
        parent_map = {}
118
        cache = self._cache
119
        for key in keys:
120
            if key in cache:
121
                value = cache[key]
122
                if value is not None:
123
                    parent_map[key] = value
124
            else:
125
                needed.add(key)
126
127
        if needed:
128
            new_parents = self._real_provider.get_parent_map(needed)
129
            cache.update(new_parents)
130
            parent_map.update(new_parents)
131
            needed.difference_update(new_parents)
132
            cache.update(dict.fromkeys(needed, None))
133
        return parent_map
134
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
135
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
136
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
137
    """Provide incremental access to revision graphs.
138
139
    This is the generic implementation; it is intended to be subclassed to
140
    specialize it for other repository types.
141
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
142
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
143
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
144
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
145
146
        This should not normally be invoked directly, because there may be
147
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
148
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
149
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
150
        :param parents_provider: An object providing a get_parent_map call
151
            conforming to the behavior of
152
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
153
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
154
        if getattr(parents_provider, 'get_parents', None) is not None:
155
            self.get_parents = parents_provider.get_parents
156
        if getattr(parents_provider, 'get_parent_map', None) is not None:
157
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
158
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
159
160
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
161
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
162
163
    def find_lca(self, *revisions):
164
        """Determine the lowest common ancestors of the provided revisions
165
166
        A lowest common ancestor is a common ancestor none of whose
167
        descendants are common ancestors.  In graphs, unlike trees, there may
168
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
169
170
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
171
        and phase 2 filters border ancestors to determine lowest common
172
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
173
174
        In phase 1, border ancestors are identified, using a breadth-first
175
        search starting at the bottom of the graph.  Searches are stopped
176
        whenever a node or one of its descendants is determined to be common
177
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
178
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
179
        common ancestors.  This is done by searching the ancestries of each
180
        border ancestor.
181
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
182
        Phase 2 is perfomed on the principle that a border ancestor that is
183
        not an ancestor of any other border ancestor is a least common
184
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
185
186
        Searches are stopped when they find a node that is determined to be a
187
        common ancestor of all border ancestors, because this shows that it
188
        cannot be a descendant of any border ancestor.
189
190
        The scaling of this operation should be proportional to
191
        1. The number of uncommon ancestors
192
        2. The number of border ancestors
193
        3. The length of the shortest path between a border ancestor and an
194
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
195
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
196
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
197
        # We may have common ancestors that can be reached from each other.
198
        # - ask for the heads of them to filter it down to only ones that
199
        # cannot be reached from each other - phase 2.
200
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
201
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
202
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
203
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
204
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
205
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
206
        self._search_for_extra_common(common, searchers)
207
        left = searchers[0].seen
208
        right = searchers[1].seen
209
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
210
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
211
    def find_unique_ancestors(self, unique_revision, common_revisions):
212
        """Find the unique ancestors for a revision versus others.
213
214
        This returns the ancestry of unique_revision, excluding all revisions
215
        in the ancestry of common_revisions. If unique_revision is in the
216
        ancestry, then the empty set will be returned.
217
218
        :param unique_revision: The revision_id whose ancestry we are
219
            interested in.
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
220
            XXX: Would this API be better if we allowed multiple revisions on
221
                 to be searched here?
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
222
        :param common_revisions: Revision_ids of ancestries to exclude.
223
        :return: A set of revisions in the ancestry of unique_revision
224
        """
225
        if unique_revision in common_revisions:
226
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
227
228
        # Algorithm description
229
        # 1) Walk backwards from the unique node and all common nodes.
230
        # 2) When a node is seen by both sides, stop searching it in the unique
231
        #    walker, include it in the common walker.
232
        # 3) Stop searching when there are no nodes left for the unique walker.
233
        #    At this point, you have a maximal set of unique nodes. Some of
234
        #    them may actually be common, and you haven't reached them yet.
235
        # 4) Start new searchers for the unique nodes, seeded with the
236
        #    information you have so far.
237
        # 5) Continue searching, stopping the common searches when the search
238
        #    tip is an ancestor of all unique nodes.
239
        # 6) Search is done when all common searchers have completed.
240
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
241
        if 'graph' in debug.debug_flags:
242
            _mutter = trace.mutter
243
        else:
244
            def _mutter(*args, **kwargs):
245
                pass
246
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
247
        unique_searcher = self._make_breadth_first_searcher([unique_revision])
3377.3.27 by John Arbash Meinel
some simple updates
248
        # we know that unique_revision isn't in common_revisions
249
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
250
        common_searcher = self._make_breadth_first_searcher(common_revisions)
251
3377.3.27 by John Arbash Meinel
some simple updates
252
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
253
            next_unique_nodes = set(unique_searcher.step())
254
            next_common_nodes = set(common_searcher.step())
255
256
            # Check if either searcher encounters new nodes seen by the other
257
            # side.
258
            unique_are_common_nodes = next_unique_nodes.intersection(
259
                common_searcher.seen)
260
            unique_are_common_nodes.update(
261
                next_common_nodes.intersection(unique_searcher.seen))
262
            if unique_are_common_nodes:
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
263
                ancestors = unique_searcher.find_seen_ancestors(
264
                                unique_are_common_nodes)
265
                ancestors.update(common_searcher.find_seen_ancestors(ancestors))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
266
                unique_searcher.stop_searching_any(ancestors)
267
                common_searcher.start_searching(ancestors)
268
269
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
270
        if not unique_nodes:
271
            return unique_nodes
272
        unique_tips = self._remove_simple_descendants(unique_nodes,
273
                        self.get_parent_map(unique_nodes))
274
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
275
        if len(unique_tips) == 1:
276
            unique_searchers = []
277
            ancestor_all_unique = unique_searcher.find_seen_ancestors(unique_tips)
278
        else:
279
            unique_searchers = []
280
            for tip in unique_tips:
281
                revs_to_search = unique_searcher.find_seen_ancestors([tip])
282
                searcher = self._make_breadth_first_searcher(revs_to_search)
283
                # We don't care about the starting nodes.
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
284
                searcher._label = tip
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
285
                searcher.step()
286
                unique_searchers.append(searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
287
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
288
            ancestor_all_unique = None
289
            for searcher in unique_searchers:
290
                if ancestor_all_unique is None:
291
                    ancestor_all_unique = set(searcher.seen)
292
                else:
293
                    ancestor_all_unique = ancestor_all_unique.intersection(
294
                                                searcher.seen)
295
        # Collapse all the common nodes into a single searcher
296
        all_unique_searcher = self._make_breadth_first_searcher(ancestor_all_unique)
3377.3.36 by John Arbash Meinel
Small updates, try to write a test for the race condition.
297
        if ancestor_all_unique:
298
            all_unique_searcher.step()
299
300
            # Stop any search tips that are already known as ancestors of the
301
            # unique nodes
302
            common_searcher.stop_searching_any(
303
                common_searcher.find_seen_ancestors(ancestor_all_unique))
304
305
            total_stopped = 0
306
            for searcher in unique_searchers:
307
                total_stopped += len(searcher.stop_searching_any(
308
                    searcher.find_seen_ancestors(ancestor_all_unique)))
309
            _mutter('For %s unique nodes, created %s + 1 unique searchers'
310
                    ' (%s stopped search tips, %s common ancestors)',
311
                    len(unique_nodes), len(unique_searchers), total_stopped,
312
                    len(ancestor_all_unique))
313
            del ancestor_all_unique
3377.3.32 by John Arbash Meinel
try collapsing the common revisions into a single searcher.
314
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
315
        # While we still have common nodes to search
316
        while common_searcher._next_query:
317
            newly_seen_common = set(common_searcher.step())
318
            newly_seen_unique = set()
319
            for searcher in unique_searchers:
320
                newly_seen_unique.update(searcher.step())
321
            # These nodes are common ancestors of all unique nodes
322
            unique_are_common_nodes = newly_seen_unique.copy()
323
            for searcher in unique_searchers:
324
                unique_are_common_nodes = unique_are_common_nodes.intersection(
325
                                            searcher.seen)
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
326
            unique_are_common_nodes = unique_are_common_nodes.intersection(
327
                                        all_unique_searcher.seen)
3377.3.32 by John Arbash Meinel
try collapsing the common revisions into a single searcher.
328
            unique_are_common_nodes.update(all_unique_searcher.step())
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
329
            if newly_seen_common:
330
                # If a 'common' node is an ancestor of all unique searchers, we
331
                # can stop searching it.
332
                common_searcher.stop_searching_any(
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
333
                    all_unique_searcher.seen.intersection(newly_seen_common))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
334
            if unique_are_common_nodes:
335
                # We have new common-to-all-unique-searchers nodes
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
336
                for searcher in unique_searchers:
337
                    unique_are_common_nodes.update(
338
                        searcher.find_seen_ancestors(unique_are_common_nodes))
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
339
                unique_are_common_nodes.update(
340
                    all_unique_searcher.find_seen_ancestors(unique_are_common_nodes))
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
341
                # Since these are common, we can grab another set of ancestors
342
                # that we have seen
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
343
                unique_are_common_nodes.update(
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
344
                    common_searcher.find_seen_ancestors(unique_are_common_nodes))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
345
3377.3.32 by John Arbash Meinel
try collapsing the common revisions into a single searcher.
346
                # The all_unique searcher can start searching the common nodes
347
                # but everyone else can stop.
348
                all_unique_searcher.start_searching(unique_are_common_nodes)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
349
                common_searcher.stop_searching_any(unique_are_common_nodes)
350
351
                # Filter out searchers that don't actually search different
352
                # nodes. We already have the ancestry intersection for them
353
                next_unique_searchers = []
354
                unique_search_sets = set()
355
                for searcher in unique_searchers:
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
356
                    stopped = searcher.stop_searching_any(unique_are_common_nodes)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
357
                    will_search_set = frozenset(searcher._next_query)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
358
                    if not will_search_set:
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
359
                        _mutter('Unique searcher %s was stopped.'
360
                                ' (%s iterations) %d nodes stopped',
361
                                searcher._label,
362
                                searcher._iterations,
363
                                len(stopped))
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
364
                    elif will_search_set not in unique_search_sets:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
365
                        # This searcher is searching a unique set of nodes, let it
366
                        unique_search_sets.add(will_search_set)
367
                        next_unique_searchers.append(searcher)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
368
                    else:
3377.3.34 by John Arbash Meinel
When collapsing common revisions into another searcher,
369
                        _mutter('Unique searcher %s stopped for repeated'
370
                                ' search of %s nodes', 
371
                                searcher._label, len(will_search_set))
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
372
                if len(unique_searchers) != len(next_unique_searchers):
373
                    _mutter('Collapsed %s unique searchers => %s'
374
                            ' at %s iterations',
375
                            len(unique_searchers), len(next_unique_searchers),
376
                            all_unique_searcher._iterations)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
377
                unique_searchers = next_unique_searchers
378
        return unique_nodes.difference(common_searcher.seen)
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
379
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
380
    @symbol_versioning.deprecated_method(symbol_versioning.one_one)
381
    def get_parents(self, revisions):
382
        """Find revision ids of the parents of a list of revisions
383
384
        A list is returned of the same length as the input.  Each entry
385
        is a list of parent ids for the corresponding input revision.
386
387
        [NULL_REVISION] is used as the parent of the first user-committed
388
        revision.  Its parent list is empty.
389
390
        If the revision is not present (i.e. a ghost), None is used in place
391
        of the list of parents.
392
393
        Deprecated in bzr 1.2 - please see get_parent_map.
394
        """
395
        parents = self.get_parent_map(revisions)
3377.3.5 by John Arbash Meinel
Fix a latent bug in Graph.get_parents()
396
        return [parents.get(r, None) for r in revisions]
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
397
398
    def get_parent_map(self, revisions):
399
        """Get a map of key:parent_list for revisions.
400
401
        This implementation delegates to get_parents, for old parent_providers
402
        that do not supply get_parent_map.
403
        """
404
        result = {}
405
        for rev, parents in self.get_parents(revisions):
406
            if parents is not None:
407
                result[rev] = parents
408
        return result
409
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
410
    def _make_breadth_first_searcher(self, revisions):
411
        return _BreadthFirstSearcher(revisions, self)
412
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
413
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
414
        """Find common ancestors with at least one uncommon descendant.
415
416
        Border ancestors are identified using a breadth-first
417
        search starting at the bottom of the graph.  Searches are stopped
418
        whenever a node or one of its descendants is determined to be common.
419
420
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
421
422
        As well as the border ancestors, a set of seen common ancestors and a
423
        list of sets of seen ancestors for each input revision is returned.
424
        This allows calculation of graph difference from the results of this
425
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
426
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
427
        if None in revisions:
428
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
429
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
430
        searchers = [self._make_breadth_first_searcher([r])
431
                     for r in revisions]
432
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
433
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
434
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
435
        while True:
436
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
437
            for searcher in searchers:
438
                new_ancestors = searcher.step()
439
                if new_ancestors:
440
                    newly_seen.update(new_ancestors)
441
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
442
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
443
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
444
                    # Not a border ancestor because it was seen as common
445
                    # already
446
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
447
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
448
                for searcher in searchers:
449
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
450
                        break
451
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
452
                    # This is a border because it is a first common that we see
453
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
454
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
455
                    new_common.add(revision)
456
            if new_common:
457
                for searcher in searchers:
458
                    new_common.update(searcher.find_seen_ancestors(new_common))
459
                for searcher in searchers:
460
                    searcher.start_searching(new_common)
461
                common_ancestors.update(new_common)
462
463
            # Figure out what the searchers will be searching next, and if
464
            # there is only 1 set being searched, then we are done searching,
465
            # since all searchers would have to be searching the same data,
466
            # thus it *must* be in common.
467
            unique_search_sets = set()
468
            for searcher in searchers:
469
                will_search_set = frozenset(searcher._next_query)
470
                if will_search_set not in unique_search_sets:
471
                    # This searcher is searching a unique set of nodes, let it
472
                    unique_search_sets.add(will_search_set)
473
474
            if len(unique_search_sets) == 1:
475
                nodes = unique_search_sets.pop()
476
                uncommon_nodes = nodes.difference(common_ancestors)
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
477
                if uncommon_nodes:
478
                    raise AssertionError("Somehow we ended up converging"
479
                                         " without actually marking them as"
480
                                         " in common."
481
                                         "\nStart_nodes: %s"
482
                                         "\nuncommon_nodes: %s"
483
                                         % (revisions, uncommon_nodes))
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
484
                break
485
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
486
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
487
    def heads(self, keys):
488
        """Return the heads from amongst keys.
489
490
        This is done by searching the ancestries of each key.  Any key that is
491
        reachable from another key is not returned; all the others are.
492
493
        This operation scales with the relative depth between any two keys. If
494
        any two keys are completely disconnected all ancestry of both sides
495
        will be retrieved.
496
497
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
498
        :return: A set of the heads. Note that as a set there is no ordering
499
            information. Callers will need to filter their input to create
500
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
501
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
502
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
503
        if revision.NULL_REVISION in candidate_heads:
504
            # NULL_REVISION is only a head if it is the only entry
505
            candidate_heads.remove(revision.NULL_REVISION)
506
            if not candidate_heads:
507
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
508
        if len(candidate_heads) < 2:
509
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
510
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
511
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
512
        active_searchers = dict(searchers)
513
        # skip over the actual candidate for each searcher
514
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
515
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
516
        # The common walker finds nodes that are common to two or more of the
517
        # input keys, so that we don't access all history when a currently
518
        # uncommon search point actually meets up with something behind a
519
        # common search point. Common search points do not keep searches
520
        # active; they just allow us to make searches inactive without
521
        # accessing all history.
522
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
523
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
524
            ancestors = set()
525
            # advance searches
526
            try:
527
                common_walker.next()
528
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
529
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
530
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
531
            for candidate in active_searchers.keys():
532
                try:
533
                    searcher = active_searchers[candidate]
534
                except KeyError:
535
                    # rare case: we deleted candidate in a previous iteration
536
                    # through this for loop, because it was determined to be
537
                    # a descendant of another candidate.
538
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
539
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
540
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
541
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
542
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
543
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
544
            # process found nodes
545
            new_common = set()
546
            for ancestor in ancestors:
547
                if ancestor in candidate_heads:
548
                    candidate_heads.remove(ancestor)
549
                    del searchers[ancestor]
550
                    if ancestor in active_searchers:
551
                        del active_searchers[ancestor]
552
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
553
                if ancestor in common_walker.seen:
554
                    # some searcher has encountered our known common nodes:
555
                    # just stop it
556
                    ancestor_set = set([ancestor])
557
                    for searcher in searchers.itervalues():
558
                        searcher.stop_searching_any(ancestor_set)
559
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
560
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
561
                    for searcher in searchers.itervalues():
562
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
563
                            break
564
                    else:
2921.3.4 by Robert Collins
Review feedback.
565
                        # The final active searcher has just reached this node,
566
                        # making it be known as a descendant of all candidates,
567
                        # so we can stop searching it, and any seen ancestors
568
                        new_common.add(ancestor)
569
                        for searcher in searchers.itervalues():
570
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
571
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
572
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
573
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
574
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
575
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
576
    def find_unique_lca(self, left_revision, right_revision,
577
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
578
        """Find a unique LCA.
579
580
        Find lowest common ancestors.  If there is no unique  common
581
        ancestor, find the lowest common ancestors of those ancestors.
582
583
        Iteration stops when a unique lowest common ancestor is found.
584
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
585
586
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
587
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
588
589
        :param count_steps: If True, the return value will be a tuple of
590
            (unique_lca, steps) where steps is the number of times that
591
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
592
        """
593
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
594
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
595
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
596
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
597
            lca = self.find_lca(*revisions)
598
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
599
                result = lca.pop()
600
                if count_steps:
601
                    return result, steps
602
                else:
603
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
604
            if len(lca) == 0:
605
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
606
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
607
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
608
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
609
        """Iterate the ancestry of this revision.
610
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
611
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
612
        :return: Yield tuples mapping a revision_id to its parents for the
613
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
614
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
615
            with no parents will have NULL_REVISION as their only parent. (As
616
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
617
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
618
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
619
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
620
        processed = set()
621
        while pending:
622
            processed.update(pending)
623
            next_map = self.get_parent_map(pending)
624
            next_pending = set()
625
            for item in next_map.iteritems():
626
                yield item
627
                next_pending.update(p for p in item[1] if p not in processed)
628
            ghosts = pending.difference(next_map)
629
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
630
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
631
            pending = next_pending
632
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
633
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
634
        """Iterate through the input revisions in topological order.
635
636
        This sorting only ensures that parents come before their children.
637
        An ancestor may sort after a descendant if the relationship is not
638
        visible in the supplied list of revisions.
639
        """
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
640
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
641
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
642
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
643
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
644
        """Determine whether a revision is an ancestor of another.
645
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
646
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
647
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
648
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
649
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
650
        return set([candidate_descendant]) == self.heads(
651
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
652
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
653
    def _search_for_extra_common(self, common, searchers):
654
        """Make sure that unique nodes are genuinely unique.
655
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
656
        After _find_border_ancestors, all nodes marked "common" are indeed
657
        common. Some of the nodes considered unique are not, due to history
658
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
659
660
        We know that we have searched enough when all common search tips are
661
        descended from all unique (uncommon) nodes because we know that a node
662
        cannot be an ancestor of its own ancestor.
663
664
        :param common: A set of common nodes
665
        :param searchers: The searchers returned from _find_border_ancestors
666
        :return: None
667
        """
668
        # Basic algorithm...
669
        #   A) The passed in searchers should all be on the same tips, thus
670
        #      they should be considered the "common" searchers.
671
        #   B) We find the difference between the searchers, these are the
672
        #      "unique" nodes for each side.
673
        #   C) We do a quick culling so that we only start searching from the
674
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
675
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
676
        #   D) We start searching for ancestors common to all unique nodes.
677
        #   E) We have the common searchers stop searching any ancestors of
678
        #      nodes found by (D)
679
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
680
681
        # TODO: We need a way to remove unique_searchers when they overlap with
682
        #       other unique searchers.
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
683
        if len(searchers) != 2:
684
            raise NotImplementedError(
685
                "Algorithm not yet implemented for > 2 searchers")
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
686
        common_searchers = searchers
687
        left_searcher = searchers[0]
688
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
689
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
690
        if not unique: # No unique nodes, nothing to do
691
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
692
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
693
        unique = self._remove_simple_descendants(unique,
694
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
695
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
696
697
        unique_searchers = []
698
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
699
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
700
                parent_searcher = left_searcher
701
            else:
702
                parent_searcher = right_searcher
703
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
704
            if not revs_to_search: # XXX: This shouldn't be possible
705
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
706
            searcher = self._make_breadth_first_searcher(revs_to_search)
707
            # We don't care about the starting nodes.
708
            searcher.step()
709
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
710
3377.3.16 by John Arbash Meinel
small cleanups
711
        # possible todo: aggregate the common searchers into a single common
712
        #   searcher, just make sure that we include the nodes into the .seen
713
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
714
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
715
        ancestor_all_unique = None
716
        for searcher in unique_searchers:
717
            if ancestor_all_unique is None:
718
                ancestor_all_unique = set(searcher.seen)
719
            else:
720
                ancestor_all_unique = ancestor_all_unique.intersection(
721
                                            searcher.seen)
722
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
723
        trace.mutter('Started %s unique searchers for %s unique revisions',
724
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
725
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
726
        while True: # If we have no more nodes we have nothing to do
727
            newly_seen_common = set()
728
            for searcher in common_searchers:
729
                newly_seen_common.update(searcher.step())
730
            newly_seen_unique = set()
731
            for searcher in unique_searchers:
732
                newly_seen_unique.update(searcher.step())
733
            new_common_unique = set()
734
            for revision in newly_seen_unique:
735
                for searcher in unique_searchers:
736
                    if revision not in searcher.seen:
737
                        break
738
                else:
739
                    # This is a border because it is a first common that we see
740
                    # after walking for a while.
741
                    new_common_unique.add(revision)
742
            if newly_seen_common:
743
                # These are nodes descended from one of the 'common' searchers.
744
                # Make sure all searchers are on the same page
745
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
746
                    newly_seen_common.update(
747
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
748
                # We start searching the whole ancestry. It is a bit wasteful,
749
                # though. We really just want to mark all of these nodes as
750
                # 'seen' and then start just the tips. However, it requires a
751
                # get_parent_map() call to figure out the tips anyway, and all
752
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
753
                for searcher in common_searchers:
754
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
755
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
756
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
757
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
758
                stop_searching_common = ancestor_all_unique.intersection(
759
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
760
                if stop_searching_common:
761
                    for searcher in common_searchers:
762
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
763
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
764
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
765
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
766
                    new_common_unique.update(
767
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
768
                # Since these are common, we can grab another set of ancestors
769
                # that we have seen
770
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
771
                    new_common_unique.update(
772
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
773
774
                # We can tell all of the unique searchers to start at these
775
                # nodes, and tell all of the common searchers to *stop*
776
                # searching these nodes
777
                for searcher in unique_searchers:
778
                    searcher.start_searching(new_common_unique)
779
                for searcher in common_searchers:
780
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
781
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
782
3377.3.20 by John Arbash Meinel
comment cleanups.
783
                # Filter out searchers that don't actually search different
784
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
785
                next_unique_searchers = []
786
                unique_search_sets = set()
787
                for searcher in unique_searchers:
788
                    will_search_set = frozenset(searcher._next_query)
789
                    if will_search_set not in unique_search_sets:
790
                        # This searcher is searching a unique set of nodes, let it
791
                        unique_search_sets.add(will_search_set)
792
                        next_unique_searchers.append(searcher)
793
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
794
            for searcher in common_searchers:
795
                if searcher._next_query:
796
                    break
797
            else:
798
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
799
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
800
801
    def _remove_simple_descendants(self, revisions, parent_map):
802
        """remove revisions which are children of other ones in the set
803
804
        This doesn't do any graph searching, it just checks the immediate
805
        parent_map to find if there are any children which can be removed.
806
807
        :param revisions: A set of revision_ids
808
        :return: A set of revision_ids with the children removed
809
        """
810
        simple_ancestors = revisions.copy()
811
        # TODO: jam 20071214 we *could* restrict it to searching only the
812
        #       parent_map of revisions already present in 'revisions', but
813
        #       considering the general use case, I think this is actually
814
        #       better.
815
816
        # This is the same as the following loop. I don't know that it is any
817
        # faster.
818
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
819
        ##     if p_ids is not None and revisions.intersection(p_ids))
820
        ## return simple_ancestors
821
822
        # Yet Another Way, invert the parent map (which can be cached)
823
        ## descendants = {}
824
        ## for revision_id, parent_ids in parent_map.iteritems():
825
        ##   for p_id in parent_ids:
826
        ##       descendants.setdefault(p_id, []).append(revision_id)
827
        ## for revision in revisions.intersection(descendants):
828
        ##   simple_ancestors.difference_update(descendants[revision])
829
        ## return simple_ancestors
830
        for revision, parent_ids in parent_map.iteritems():
831
            if parent_ids is None:
832
                continue
833
            for parent_id in parent_ids:
834
                if parent_id in revisions:
835
                    # This node has a parent present in the set, so we can
836
                    # remove it
837
                    simple_ancestors.discard(revision)
838
                    break
839
        return simple_ancestors
840
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
841
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
842
class HeadsCache(object):
843
    """A cache of results for graph heads calls."""
844
845
    def __init__(self, graph):
846
        self.graph = graph
847
        self._heads = {}
848
849
    def heads(self, keys):
850
        """Return the heads of keys.
851
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
852
        This matches the API of Graph.heads(), specifically the return value is
853
        a set which can be mutated, and ordering of the input is not preserved
854
        in the output.
855
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
856
        :see also: Graph.heads.
857
        :param keys: The keys to calculate heads for.
858
        :return: A set containing the heads, which may be mutated without
859
            affecting future lookups.
860
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
861
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
862
        try:
863
            return set(self._heads[keys])
864
        except KeyError:
865
            heads = self.graph.heads(keys)
866
            self._heads[keys] = heads
867
            return set(heads)
868
869
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
870
class FrozenHeadsCache(object):
871
    """Cache heads() calls, assuming the caller won't modify them."""
872
873
    def __init__(self, graph):
874
        self.graph = graph
875
        self._heads = {}
876
877
    def heads(self, keys):
878
        """Return the heads of keys.
879
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
880
        Similar to Graph.heads(). The main difference is that the return value
881
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
882
883
        :see also: Graph.heads.
884
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
885
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
886
        """
887
        keys = frozenset(keys)
888
        try:
889
            return self._heads[keys]
890
        except KeyError:
891
            heads = frozenset(self.graph.heads(keys))
892
            self._heads[keys] = heads
893
            return heads
894
895
    def cache(self, keys, heads):
896
        """Store a known value."""
897
        self._heads[frozenset(keys)] = frozenset(heads)
898
899
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
900
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
901
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
902
903
    This class implements the iterator protocol, but additionally
904
    1. provides a set of seen ancestors, and
905
    2. allows some ancestries to be unsearched, via stop_searching_any
906
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
907
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
908
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
909
        self._iterations = 0
910
        self._next_query = set(revisions)
911
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
912
        self._started_keys = set(self._next_query)
913
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
914
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
915
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
916
        self._current_present = set()
917
        self._current_ghosts = set()
918
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
919
920
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
921
        if self._iterations:
922
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
923
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
924
            prefix = "starting"
925
        search = '%s=%r' % (prefix, list(self._next_query))
926
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
927
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
928
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
929
    def get_result(self):
930
        """Get a SearchResult for the current state of this searcher.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
931
        
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
932
        :return: A SearchResult for this search so far. The SearchResult is
933
            static - the search can be advanced and the search result will not
934
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
935
        """
936
        if self._returning == 'next':
937
            # We have to know the current nodes children to be able to list the
938
            # exclude keys for them. However, while we could have a second
939
            # look-ahead result buffer and shuffle things around, this method
940
            # is typically only called once per search - when memoising the
3211.5.1 by Robert Collins
Change the smart server get_parents method to take a graph search to exclude already recieved parents from. This prevents history shortcuts causing huge numbers of duplicates.
941
            # results of the search. 
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
942
            found, ghosts, next, parents = self._do_query(self._next_query)
943
            # pretend we didn't query: perhaps we should tweak _do_query to be
944
            # entirely stateless?
945
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
946
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
947
        else:
948
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
949
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
950
        included_keys = self.seen.difference(excludes)
951
        return SearchResult(self._started_keys, excludes, len(included_keys),
952
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
953
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
954
    def step(self):
955
        try:
956
            return self.next()
957
        except StopIteration:
958
            return ()
959
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
960
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
961
        """Return the next ancestors of this revision.
962
2490.2.12 by Aaron Bentley
Improve documentation
963
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
964
        traversal.  No ancestor will be returned more than once. Ancestors are
965
        returned before their parentage is queried, so ghosts and missing
966
        revisions (including the start revisions) are included in the result.
967
        This can save a round trip in LCA style calculation by allowing
968
        convergence to be detected without reading the data for the revision
969
        the convergence occurs on.
970
971
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
972
        """
3177.3.3 by Robert Collins
Review feedback.
973
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
974
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
975
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
976
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
977
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
978
            self._advance()
979
        if len(self._next_query) == 0:
980
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
981
        # We have seen what we're querying at this point as we are returning
982
        # the query, not the results.
983
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
984
        return self._next_query
985
986
    def next_with_ghosts(self):
987
        """Return the next found ancestors, with ghosts split out.
988
        
989
        Ancestors are returned in the order they are seen in a breadth-first
990
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
991
        returned only after asking for their parents, which allows us to detect
992
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
993
994
        :return: A tuple with (present ancestors, ghost ancestors) sets.
995
        """
3177.3.3 by Robert Collins
Review feedback.
996
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
997
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
998
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
999
            self._advance()
1000
        if len(self._next_query) == 0:
1001
            raise StopIteration()
1002
        self._advance()
1003
        return self._current_present, self._current_ghosts
1004
1005
    def _advance(self):
1006
        """Advance the search.
1007
1008
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1009
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1010
        """
1011
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1012
        found, ghosts, next, parents = self._do_query(self._next_query)
1013
        self._current_present = found
1014
        self._current_ghosts = ghosts
1015
        self._next_query = next
1016
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1017
        # ghosts are implicit stop points, otherwise the search cannot be
1018
        # repeated when ghosts are filled.
1019
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1020
1021
    def _do_query(self, revisions):
1022
        """Query for revisions.
1023
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1024
        Adds revisions to the seen set.
1025
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1026
        :param revisions: Revisions to query.
1027
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1028
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1029
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1030
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1031
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1032
        # revisions may contain nodes that point to other nodes in revisions:
1033
        # we want to filter them out.
1034
        self.seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1035
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1036
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1037
        for rev_id, parents in parent_map.iteritems():
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1038
            new_found_parents = [p for p in parents if p not in self.seen]
1039
            if new_found_parents:
1040
                # Calling set.update() with an empty generator is actually
1041
                # rather expensive.
1042
                parents_of_found.update(new_found_parents)
1043
        ghost_revisions = revisions - found_revisions
1044
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1045
2490.2.8 by Aaron Bentley
fix iteration stuff
1046
    def __iter__(self):
1047
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1048
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1049
    def find_seen_ancestors(self, revisions):
1050
        """Find ancestors of these revisions that have already been seen."""
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1051
        all_seen = self.seen
1052
        pending = set(revisions).intersection(all_seen)
1053
        seen_ancestors = set(pending)
1054
1055
        if self._returning == 'next':
1056
            # self.seen contains what nodes have been returned, not what nodes
1057
            # have been queried. We don't want to probe for nodes that haven't
1058
            # been searched yet.
1059
            not_searched_yet = self._next_query
1060
        else:
1061
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1062
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1063
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1064
        while pending:
1065
            parent_map = get_parent_map(pending)
1066
            all_parents = []
1067
            # We don't care if it is a ghost, since it can't be seen if it is
1068
            # a ghost
1069
            for parent_ids in parent_map.itervalues():
1070
                all_parents.extend(parent_ids)
1071
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1072
            seen_ancestors.update(next_pending)
1073
            next_pending.difference_update(not_searched_yet)
1074
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1075
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1076
        return seen_ancestors
1077
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1078
    def stop_searching_any(self, revisions):
1079
        """
1080
        Remove any of the specified revisions from the search list.
1081
1082
        None of the specified revisions are required to be present in the
2490.2.12 by Aaron Bentley
Improve documentation
1083
        search list.  In this case, the call is a no-op.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1084
        """
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1085
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1086
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1087
            stopped = self._next_query.intersection(revisions)
1088
            self._next_query = self._next_query.difference(revisions)
1089
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1090
            stopped_present = self._current_present.intersection(revisions)
1091
            stopped = stopped_present.union(
1092
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1093
            self._current_present.difference_update(stopped)
1094
            self._current_ghosts.difference_update(stopped)
1095
            # stopping 'x' should stop returning parents of 'x', but 
1096
            # not if 'y' always references those same parents
1097
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1098
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1099
                for parent_id in self._current_parents[rev]:
1100
                    if parent_id not in stop_rev_references:
1101
                        stop_rev_references[parent_id] = 0
1102
                    stop_rev_references[parent_id] += 1
1103
            # if only the stopped revisions reference it, the ref count will be
1104
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1105
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1106
                for parent_id in parents:
1107
                    try:
1108
                        stop_rev_references[parent_id] -= 1
1109
                    except KeyError:
1110
                        pass
1111
            stop_parents = set()
1112
            for rev_id, refs in stop_rev_references.iteritems():
1113
                if refs == 0:
1114
                    stop_parents.add(rev_id)
1115
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1116
        self._stopped_keys.update(stopped)
2490.2.25 by Aaron Bentley
Update from review
1117
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1118
1119
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1120
        """Add revisions to the search.
1121
1122
        The parents of revisions will be returned from the next call to next()
1123
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1124
        next* call then the return value is the result of looking up the
1125
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1126
        """
1127
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1128
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1129
        new_revisions = revisions.difference(self.seen)
3177.3.3 by Robert Collins
Review feedback.
1130
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1131
            self._next_query.update(new_revisions)
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1132
            self.seen.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1133
        else:
1134
            # perform a query on revisions
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1135
            revs, ghosts, query, parents = self._do_query(revisions)
1136
            self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1137
            self._current_present.update(revs)
1138
            self._current_ghosts.update(ghosts)
1139
            self._next_query.update(query)
1140
            self._current_parents.update(parents)
1141
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1142
1143
1144
class SearchResult(object):
1145
    """The result of a breadth first search.
1146
1147
    A SearchResult provides the ability to reconstruct the search or access a
1148
    set of the keys the search found.
1149
    """
1150
1151
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1152
        """Create a SearchResult.
1153
1154
        :param start_keys: The keys the search started at.
1155
        :param exclude_keys: The keys the search excludes.
1156
        :param key_count: The total number of keys (from start to but not
1157
            including exclude).
1158
        :param keys: The keys the search found. Note that in future we may get
1159
            a SearchResult from a smart server, in which case the keys list is
1160
            not necessarily immediately available.
1161
        """
1162
        self._recipe = (start_keys, exclude_keys, key_count)
1163
        self._keys = frozenset(keys)
1164
1165
    def get_recipe(self):
1166
        """Return a recipe that can be used to replay this search.
1167
        
1168
        The recipe allows reconstruction of the same results at a later date
1169
        without knowing all the found keys. The essential elements are a list
1170
        of keys to start and and to stop at. In order to give reproducible
1171
        results when ghosts are encountered by a search they are automatically
1172
        added to the exclude list (or else ghost filling may alter the
1173
        results).
1174
1175
        :return: A tuple (start_keys_set, exclude_keys_set, revision_count). To
1176
            recreate the results of this search, create a breadth first
1177
            searcher on the same graph starting at start_keys. Then call next()
1178
            (or next_with_ghosts()) repeatedly, and on every result, call
1179
            stop_searching_any on any keys from the exclude_keys set. The
1180
            revision_count value acts as a trivial cross-check - the found
1181
            revisions of the new search should have as many elements as
1182
            revision_count. If it does not, then additional revisions have been
1183
            ghosted since the search was executed the first time and the second
1184
            time.
1185
        """
1186
        return self._recipe
1187
1188
    def get_keys(self):
1189
        """Return the keys found in this search.
1190
1191
        :return: A set of keys.
1192
        """
1193
        return self._keys
1194