/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
17
import time
18
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
19
from bzrlib import (
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
20
    debug,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
21
    errors,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
22
    revision,
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
23
    symbol_versioning,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
24
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
25
    tsort,
26
    )
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
27
from bzrlib.deprecated_graph import (node_distances, select_farthest)
2490.2.1 by Aaron Bentley
Start work on GraphWalker
28
3377.4.9 by John Arbash Meinel
STEP every 5
29
STEP_UNIQUE_SEARCHER_EVERY = 5
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
30
2490.2.25 by Aaron Bentley
Update from review
31
# DIAGRAM of terminology
32
#       A
33
#       /\
34
#      B  C
35
#      |  |\
36
#      D  E F
37
#      |\/| |
38
#      |/\|/
39
#      G  H
40
#
41
# In this diagram, relative to G and H:
42
# A, B, C, D, E are common ancestors.
43
# C, D and E are border ancestors, because each has a non-common descendant.
44
# D and E are least common ancestors because none of their descendants are
45
# common ancestors.
46
# C is not a least common ancestor because its descendant, E, is a common
47
# ancestor.
48
#
49
# The find_unique_lca algorithm will pick A in two steps:
50
# 1. find_lca('G', 'H') => ['D', 'E']
51
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
52
53
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
54
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
55
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
56
57
    def __init__(self, ancestry):
58
        self.ancestry = ancestry
59
60
    def __repr__(self):
61
        return 'DictParentsProvider(%r)' % self.ancestry
62
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
63
    def get_parent_map(self, keys):
64
        """See _StackedParentsProvider.get_parent_map"""
65
        ancestry = self.ancestry
66
        return dict((k, ancestry[k]) for k in keys if k in ancestry)
67
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
68
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
69
class _StackedParentsProvider(object):
70
71
    def __init__(self, parent_providers):
72
        self._parent_providers = parent_providers
73
2490.2.28 by Aaron Bentley
Fix handling of null revision
74
    def __repr__(self):
75
        return "_StackedParentsProvider(%r)" % self._parent_providers
76
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
77
    def get_parent_map(self, keys):
78
        """Get a mapping of keys => parents
79
80
        A dictionary is returned with an entry for each key present in this
81
        source. If this source doesn't have information about a key, it should
82
        not include an entry.
83
84
        [NULL_REVISION] is used as the parent of the first user-committed
85
        revision.  Its parent list is empty.
86
87
        :param keys: An iterable returning keys to check (eg revision_ids)
88
        :return: A dictionary mapping each key to its parents
89
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
90
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
91
        remaining = set(keys)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
92
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
93
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
94
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
95
            remaining.difference_update(new_found)
96
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
97
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
98
        return found
99
100
101
class CachingParentsProvider(object):
102
    """A parents provider which will cache the revision => parents in a dict.
103
104
    This is useful for providers that have an expensive lookup.
105
    """
106
107
    def __init__(self, parent_provider):
108
        self._real_provider = parent_provider
109
        # Theoretically we could use an LRUCache here
110
        self._cache = {}
111
112
    def __repr__(self):
113
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
114
115
    def get_parent_map(self, keys):
116
        """See _StackedParentsProvider.get_parent_map"""
117
        needed = set()
118
        # If the _real_provider doesn't have a key, we cache a value of None,
119
        # which we then later use to realize we cannot provide a value for that
120
        # key.
121
        parent_map = {}
122
        cache = self._cache
123
        for key in keys:
124
            if key in cache:
125
                value = cache[key]
126
                if value is not None:
127
                    parent_map[key] = value
128
            else:
129
                needed.add(key)
130
131
        if needed:
132
            new_parents = self._real_provider.get_parent_map(needed)
133
            cache.update(new_parents)
134
            parent_map.update(new_parents)
135
            needed.difference_update(new_parents)
136
            cache.update(dict.fromkeys(needed, None))
137
        return parent_map
138
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
139
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
140
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
141
    """Provide incremental access to revision graphs.
142
143
    This is the generic implementation; it is intended to be subclassed to
144
    specialize it for other repository types.
145
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
146
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
147
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
148
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
149
150
        This should not normally be invoked directly, because there may be
151
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
152
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
153
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
154
        :param parents_provider: An object providing a get_parent_map call
155
            conforming to the behavior of
156
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
157
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
158
        if getattr(parents_provider, 'get_parents', None) is not None:
159
            self.get_parents = parents_provider.get_parents
160
        if getattr(parents_provider, 'get_parent_map', None) is not None:
161
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
162
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
163
164
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
165
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
166
167
    def find_lca(self, *revisions):
168
        """Determine the lowest common ancestors of the provided revisions
169
170
        A lowest common ancestor is a common ancestor none of whose
171
        descendants are common ancestors.  In graphs, unlike trees, there may
172
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
173
174
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
175
        and phase 2 filters border ancestors to determine lowest common
176
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
177
178
        In phase 1, border ancestors are identified, using a breadth-first
179
        search starting at the bottom of the graph.  Searches are stopped
180
        whenever a node or one of its descendants is determined to be common
181
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
182
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
183
        common ancestors.  This is done by searching the ancestries of each
184
        border ancestor.
185
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
186
        Phase 2 is perfomed on the principle that a border ancestor that is
187
        not an ancestor of any other border ancestor is a least common
188
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
189
190
        Searches are stopped when they find a node that is determined to be a
191
        common ancestor of all border ancestors, because this shows that it
192
        cannot be a descendant of any border ancestor.
193
194
        The scaling of this operation should be proportional to
195
        1. The number of uncommon ancestors
196
        2. The number of border ancestors
197
        3. The length of the shortest path between a border ancestor and an
198
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
199
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
200
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
201
        # We may have common ancestors that can be reached from each other.
202
        # - ask for the heads of them to filter it down to only ones that
203
        # cannot be reached from each other - phase 2.
204
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
205
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
206
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
207
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
208
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
209
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
210
        self._search_for_extra_common(common, searchers)
211
        left = searchers[0].seen
212
        right = searchers[1].seen
213
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
214
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
215
    def find_revno(self, target_revision_id, known_revision_ids):
216
        """Determine the revno for target_revision_id.
217
218
        :param target_revision_id: A revision_id which we would like to know
219
            the revno for.
220
        :param known_revision_ids: [(revision_id, revno)] A list of known
221
            revno, revision_id tuples. We'll use this to seed the search.
222
        """
223
        # Map from revision_ids to a known value for their revno
224
        known_revnos = dict(known_revision_ids)
225
        cur_tip = target_revision_id
226
        num_steps = 0
227
        NULL_REVISION = revision.NULL_REVISION
228
229
        while cur_tip != NULL_REVISION:
230
            parent_map = self.get_parent_map([cur_tip])
231
            parents = parent_map.get(cur_tip, None)
232
            if not parents: # An empty list, or None is still a ghost
233
                raise errors.GhostRevisionsHaveNoRevno(target_revision_id,
234
                                                       cur_tip)
235
            num_steps += 1
236
            cur_tip = parents[0]
237
238
        # We reached NULL_REVISION, so the total distance is num_steps + 1
239
        return num_steps
240
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
241
    def find_unique_ancestors(self, unique_revision, common_revisions):
242
        """Find the unique ancestors for a revision versus others.
243
244
        This returns the ancestry of unique_revision, excluding all revisions
245
        in the ancestry of common_revisions. If unique_revision is in the
246
        ancestry, then the empty set will be returned.
247
248
        :param unique_revision: The revision_id whose ancestry we are
249
            interested in.
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
250
            XXX: Would this API be better if we allowed multiple revisions on
251
                 to be searched here?
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
252
        :param common_revisions: Revision_ids of ancestries to exclude.
253
        :return: A set of revisions in the ancestry of unique_revision
254
        """
255
        if unique_revision in common_revisions:
256
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
257
258
        # Algorithm description
259
        # 1) Walk backwards from the unique node and all common nodes.
260
        # 2) When a node is seen by both sides, stop searching it in the unique
261
        #    walker, include it in the common walker.
262
        # 3) Stop searching when there are no nodes left for the unique walker.
263
        #    At this point, you have a maximal set of unique nodes. Some of
264
        #    them may actually be common, and you haven't reached them yet.
265
        # 4) Start new searchers for the unique nodes, seeded with the
266
        #    information you have so far.
267
        # 5) Continue searching, stopping the common searches when the search
268
        #    tip is an ancestor of all unique nodes.
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
269
        # 6) Aggregate together unique searchers when they are searching the
270
        #    same tips. When all unique searchers are searching the same node,
271
        #    stop move it to a single 'all_unique_searcher'.
272
        # 7) The 'all_unique_searcher' represents the very 'tip' of searching.
273
        #    Most of the time this produces very little important information.
274
        #    So don't step it as quickly as the other searchers.
275
        # 8) Search is done when all common searchers have completed.
276
277
        unique_searcher, common_searcher = self._find_initial_unique_nodes(
278
            [unique_revision], common_revisions)
279
280
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
281
        if not unique_nodes:
282
            return unique_nodes
283
284
        (all_unique_searcher,
285
         unique_tip_searchers) = self._make_unique_searchers(unique_nodes,
286
                                    unique_searcher, common_searcher)
287
288
        self._refine_unique_nodes(unique_searcher, all_unique_searcher,
289
                                  unique_tip_searchers, common_searcher)
290
        true_unique_nodes = unique_nodes.difference(common_searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
291
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
292
            trace.mutter('Found %d truly unique nodes out of %d',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
293
                         len(true_unique_nodes), len(unique_nodes))
294
        return true_unique_nodes
295
296
    def _find_initial_unique_nodes(self, unique_revisions, common_revisions):
297
        """Steps 1-3 of find_unique_ancestors.
298
299
        Find the maximal set of unique nodes. Some of these might actually
300
        still be common, but we are sure that there are no other unique nodes.
301
302
        :return: (unique_searcher, common_searcher)
303
        """
304
305
        unique_searcher = self._make_breadth_first_searcher(unique_revisions)
306
        # we know that unique_revisions aren't in common_revisions, so skip
307
        # past them.
3377.3.27 by John Arbash Meinel
some simple updates
308
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
309
        common_searcher = self._make_breadth_first_searcher(common_revisions)
310
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
311
        # As long as we are still finding unique nodes, keep searching
3377.3.27 by John Arbash Meinel
some simple updates
312
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
313
            next_unique_nodes = set(unique_searcher.step())
314
            next_common_nodes = set(common_searcher.step())
315
316
            # Check if either searcher encounters new nodes seen by the other
317
            # side.
318
            unique_are_common_nodes = next_unique_nodes.intersection(
319
                common_searcher.seen)
320
            unique_are_common_nodes.update(
321
                next_common_nodes.intersection(unique_searcher.seen))
322
            if unique_are_common_nodes:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
323
                ancestors = unique_searcher.find_seen_ancestors(
324
                                unique_are_common_nodes)
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
325
                # TODO: This is a bit overboard, we only really care about
326
                #       the ancestors of the tips because the rest we
327
                #       already know. This is *correct* but causes us to
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
328
                #       search too much ancestry.
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
329
                ancestors.update(common_searcher.find_seen_ancestors(ancestors))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
330
                unique_searcher.stop_searching_any(ancestors)
331
                common_searcher.start_searching(ancestors)
332
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
333
        return unique_searcher, common_searcher
334
335
    def _make_unique_searchers(self, unique_nodes, unique_searcher,
336
                               common_searcher):
337
        """Create a searcher for all the unique search tips (step 4).
338
339
        As a side effect, the common_searcher will stop searching any nodes
340
        that are ancestors of the unique searcher tips.
341
342
        :return: (all_unique_searcher, unique_tip_searchers)
343
        """
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
344
        unique_tips = self._remove_simple_descendants(unique_nodes,
345
                        self.get_parent_map(unique_nodes))
346
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
347
        if len(unique_tips) == 1:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
348
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
349
            ancestor_all_unique = unique_searcher.find_seen_ancestors(unique_tips)
350
        else:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
351
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
352
            for tip in unique_tips:
353
                revs_to_search = unique_searcher.find_seen_ancestors([tip])
354
                revs_to_search.update(
355
                    common_searcher.find_seen_ancestors(revs_to_search))
356
                searcher = self._make_breadth_first_searcher(revs_to_search)
357
                # We don't care about the starting nodes.
358
                searcher._label = tip
359
                searcher.step()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
360
                unique_tip_searchers.append(searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
361
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
362
            ancestor_all_unique = None
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
363
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
364
                if ancestor_all_unique is None:
365
                    ancestor_all_unique = set(searcher.seen)
366
                else:
367
                    ancestor_all_unique = ancestor_all_unique.intersection(
368
                                                searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
369
        # Collapse all the common nodes into a single searcher
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
370
        all_unique_searcher = self._make_breadth_first_searcher(
371
                                ancestor_all_unique)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
372
        if ancestor_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
373
            # We've seen these nodes in all the searchers, so we'll just go to
374
            # the next
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
375
            all_unique_searcher.step()
376
377
            # Stop any search tips that are already known as ancestors of the
378
            # unique nodes
379
            stopped_common = common_searcher.stop_searching_any(
380
                common_searcher.find_seen_ancestors(ancestor_all_unique))
381
382
            total_stopped = 0
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
383
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
384
                total_stopped += len(searcher.stop_searching_any(
385
                    searcher.find_seen_ancestors(ancestor_all_unique)))
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
386
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
387
            trace.mutter('For %d unique nodes, created %d + 1 unique searchers'
388
                         ' (%d stopped search tips, %d common ancestors'
389
                         ' (%d stopped common)',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
390
                         len(unique_nodes), len(unique_tip_searchers),
391
                         total_stopped, len(ancestor_all_unique),
392
                         len(stopped_common))
393
        return all_unique_searcher, unique_tip_searchers
394
395
    def _step_unique_and_common_searchers(self, common_searcher,
396
                                          unique_tip_searchers,
397
                                          unique_searcher):
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
398
        """Step all the searchers"""
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
399
        newly_seen_common = set(common_searcher.step())
400
        newly_seen_unique = set()
401
        for searcher in unique_tip_searchers:
402
            next = set(searcher.step())
403
            next.update(unique_searcher.find_seen_ancestors(next))
404
            next.update(common_searcher.find_seen_ancestors(next))
405
            for alt_searcher in unique_tip_searchers:
406
                if alt_searcher is searcher:
407
                    continue
408
                next.update(alt_searcher.find_seen_ancestors(next))
409
            searcher.start_searching(next)
410
            newly_seen_unique.update(next)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
411
        return newly_seen_common, newly_seen_unique
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
412
413
    def _find_nodes_common_to_all_unique(self, unique_tip_searchers,
414
                                         all_unique_searcher,
415
                                         newly_seen_unique, step_all_unique):
416
        """Find nodes that are common to all unique_tip_searchers.
417
418
        If it is time, step the all_unique_searcher, and add its nodes to the
419
        result.
420
        """
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
421
        common_to_all_unique_nodes = newly_seen_unique.copy()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
422
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
423
            common_to_all_unique_nodes.intersection_update(searcher.seen)
424
        common_to_all_unique_nodes.intersection_update(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
425
                                    all_unique_searcher.seen)
426
        # Step all-unique less frequently than the other searchers.
427
        # In the common case, we don't need to spider out far here, so
428
        # avoid doing extra work.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
429
        if step_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
430
            tstart = time.clock()
431
            nodes = all_unique_searcher.step()
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
432
            common_to_all_unique_nodes.update(nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
433
            if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
434
                tdelta = time.clock() - tstart
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
435
                trace.mutter('all_unique_searcher step() took %.3fs'
436
                             'for %d nodes (%d total), iteration: %s',
437
                             tdelta, len(nodes), len(all_unique_searcher.seen),
438
                             all_unique_searcher._iterations)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
439
        return common_to_all_unique_nodes
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
440
441
    def _collapse_unique_searchers(self, unique_tip_searchers,
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
442
                                   common_to_all_unique_nodes):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
443
        """Combine searchers that are searching the same tips.
444
445
        When two searchers are searching the same tips, we can stop one of the
446
        searchers. We also know that the maximal set of common ancestors is the
447
        intersection of the two original searchers.
448
449
        :return: A list of searchers that are searching unique nodes.
450
        """
451
        # Filter out searchers that don't actually search different
452
        # nodes. We already have the ancestry intersection for them
453
        unique_search_tips = {}
454
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
455
            stopped = searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
456
            will_search_set = frozenset(searcher._next_query)
457
            if not will_search_set:
458
                if 'graph' in debug.debug_flags:
459
                    trace.mutter('Unique searcher %s was stopped.'
460
                                 ' (%s iterations) %d nodes stopped',
461
                                 searcher._label,
462
                                 searcher._iterations,
463
                                 len(stopped))
464
            elif will_search_set not in unique_search_tips:
465
                # This searcher is searching a unique set of nodes, let it
466
                unique_search_tips[will_search_set] = [searcher]
467
            else:
468
                unique_search_tips[will_search_set].append(searcher)
469
        # TODO: it might be possible to collapse searchers faster when they
470
        #       only have *some* search tips in common.
471
        next_unique_searchers = []
472
        for searchers in unique_search_tips.itervalues():
473
            if len(searchers) == 1:
474
                # Searching unique tips, go for it
475
                next_unique_searchers.append(searchers[0])
476
            else:
477
                # These searchers have started searching the same tips, we
478
                # don't need them to cover the same ground. The
479
                # intersection of their ancestry won't change, so create a
480
                # new searcher, combining their histories.
481
                next_searcher = searchers[0]
482
                for searcher in searchers[1:]:
483
                    next_searcher.seen.intersection_update(searcher.seen)
484
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
485
                    trace.mutter('Combining %d searchers into a single'
486
                                 ' searcher searching %d nodes with'
487
                                 ' %d ancestry',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
488
                                 len(searchers),
489
                                 len(next_searcher._next_query),
490
                                 len(next_searcher.seen))
491
                next_unique_searchers.append(next_searcher)
492
        return next_unique_searchers
493
494
    def _refine_unique_nodes(self, unique_searcher, all_unique_searcher,
495
                             unique_tip_searchers, common_searcher):
496
        """Steps 5-8 of find_unique_ancestors.
497
        
498
        This function returns when common_searcher has stopped searching for
499
        more nodes.
500
        """
501
        # We step the ancestor_all_unique searcher only every
502
        # STEP_UNIQUE_SEARCHER_EVERY steps.
503
        step_all_unique_counter = 0
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
504
        # While we still have common nodes to search
505
        while common_searcher._next_query:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
506
            (newly_seen_common,
507
             newly_seen_unique) = self._step_unique_and_common_searchers(
508
                common_searcher, unique_tip_searchers, unique_searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
509
            # These nodes are common ancestors of all unique nodes
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
510
            common_to_all_unique_nodes = self._find_nodes_common_to_all_unique(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
511
                unique_tip_searchers, all_unique_searcher, newly_seen_unique,
512
                step_all_unique_counter==0)
513
            step_all_unique_counter = ((step_all_unique_counter + 1)
514
                                       % STEP_UNIQUE_SEARCHER_EVERY)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
515
516
            if newly_seen_common:
517
                # If a 'common' node is an ancestor of all unique searchers, we
518
                # can stop searching it.
519
                common_searcher.stop_searching_any(
520
                    all_unique_searcher.seen.intersection(newly_seen_common))
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
521
            if common_to_all_unique_nodes:
522
                common_to_all_unique_nodes.update(
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
523
                    common_searcher.find_seen_ancestors(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
524
                        common_to_all_unique_nodes))
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
525
                # The all_unique searcher can start searching the common nodes
526
                # but everyone else can stop.
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
527
                # This is the sort of thing where we would like to not have it
528
                # start_searching all of the nodes, but only mark all of them
529
                # as seen, and have it search only the actual tips. Otherwise
530
                # it is another get_parent_map() traversal for it to figure out
531
                # what we already should know.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
532
                all_unique_searcher.start_searching(common_to_all_unique_nodes)
533
                common_searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
534
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
535
            next_unique_searchers = self._collapse_unique_searchers(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
536
                unique_tip_searchers, common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
537
            if len(unique_tip_searchers) != len(next_unique_searchers):
538
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
539
                    trace.mutter('Collapsed %d unique searchers => %d'
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
540
                                 ' at %s iterations',
541
                                 len(unique_tip_searchers),
542
                                 len(next_unique_searchers),
543
                                 all_unique_searcher._iterations)
544
            unique_tip_searchers = next_unique_searchers
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
545
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
546
    @symbol_versioning.deprecated_method(symbol_versioning.one_one)
547
    def get_parents(self, revisions):
548
        """Find revision ids of the parents of a list of revisions
549
550
        A list is returned of the same length as the input.  Each entry
551
        is a list of parent ids for the corresponding input revision.
552
553
        [NULL_REVISION] is used as the parent of the first user-committed
554
        revision.  Its parent list is empty.
555
556
        If the revision is not present (i.e. a ghost), None is used in place
557
        of the list of parents.
558
559
        Deprecated in bzr 1.2 - please see get_parent_map.
560
        """
561
        parents = self.get_parent_map(revisions)
3377.3.5 by John Arbash Meinel
Fix a latent bug in Graph.get_parents()
562
        return [parents.get(r, None) for r in revisions]
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
563
564
    def get_parent_map(self, revisions):
565
        """Get a map of key:parent_list for revisions.
566
567
        This implementation delegates to get_parents, for old parent_providers
568
        that do not supply get_parent_map.
569
        """
570
        result = {}
571
        for rev, parents in self.get_parents(revisions):
572
            if parents is not None:
573
                result[rev] = parents
574
        return result
575
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
576
    def _make_breadth_first_searcher(self, revisions):
577
        return _BreadthFirstSearcher(revisions, self)
578
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
579
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
580
        """Find common ancestors with at least one uncommon descendant.
581
582
        Border ancestors are identified using a breadth-first
583
        search starting at the bottom of the graph.  Searches are stopped
584
        whenever a node or one of its descendants is determined to be common.
585
586
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
587
588
        As well as the border ancestors, a set of seen common ancestors and a
589
        list of sets of seen ancestors for each input revision is returned.
590
        This allows calculation of graph difference from the results of this
591
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
592
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
593
        if None in revisions:
594
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
595
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
596
        searchers = [self._make_breadth_first_searcher([r])
597
                     for r in revisions]
598
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
599
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
600
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
601
        while True:
602
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
603
            for searcher in searchers:
604
                new_ancestors = searcher.step()
605
                if new_ancestors:
606
                    newly_seen.update(new_ancestors)
607
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
608
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
609
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
610
                    # Not a border ancestor because it was seen as common
611
                    # already
612
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
613
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
614
                for searcher in searchers:
615
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
616
                        break
617
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
618
                    # This is a border because it is a first common that we see
619
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
620
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
621
                    new_common.add(revision)
622
            if new_common:
623
                for searcher in searchers:
624
                    new_common.update(searcher.find_seen_ancestors(new_common))
625
                for searcher in searchers:
626
                    searcher.start_searching(new_common)
627
                common_ancestors.update(new_common)
628
629
            # Figure out what the searchers will be searching next, and if
630
            # there is only 1 set being searched, then we are done searching,
631
            # since all searchers would have to be searching the same data,
632
            # thus it *must* be in common.
633
            unique_search_sets = set()
634
            for searcher in searchers:
635
                will_search_set = frozenset(searcher._next_query)
636
                if will_search_set not in unique_search_sets:
637
                    # This searcher is searching a unique set of nodes, let it
638
                    unique_search_sets.add(will_search_set)
639
640
            if len(unique_search_sets) == 1:
641
                nodes = unique_search_sets.pop()
642
                uncommon_nodes = nodes.difference(common_ancestors)
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
643
                if uncommon_nodes:
644
                    raise AssertionError("Somehow we ended up converging"
645
                                         " without actually marking them as"
646
                                         " in common."
647
                                         "\nStart_nodes: %s"
648
                                         "\nuncommon_nodes: %s"
649
                                         % (revisions, uncommon_nodes))
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
650
                break
651
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
652
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
653
    def heads(self, keys):
654
        """Return the heads from amongst keys.
655
656
        This is done by searching the ancestries of each key.  Any key that is
657
        reachable from another key is not returned; all the others are.
658
659
        This operation scales with the relative depth between any two keys. If
660
        any two keys are completely disconnected all ancestry of both sides
661
        will be retrieved.
662
663
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
664
        :return: A set of the heads. Note that as a set there is no ordering
665
            information. Callers will need to filter their input to create
666
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
667
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
668
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
669
        if revision.NULL_REVISION in candidate_heads:
670
            # NULL_REVISION is only a head if it is the only entry
671
            candidate_heads.remove(revision.NULL_REVISION)
672
            if not candidate_heads:
673
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
674
        if len(candidate_heads) < 2:
675
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
676
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
677
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
678
        active_searchers = dict(searchers)
679
        # skip over the actual candidate for each searcher
680
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
681
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
682
        # The common walker finds nodes that are common to two or more of the
683
        # input keys, so that we don't access all history when a currently
684
        # uncommon search point actually meets up with something behind a
685
        # common search point. Common search points do not keep searches
686
        # active; they just allow us to make searches inactive without
687
        # accessing all history.
688
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
689
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
690
            ancestors = set()
691
            # advance searches
692
            try:
693
                common_walker.next()
694
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
695
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
696
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
697
            for candidate in active_searchers.keys():
698
                try:
699
                    searcher = active_searchers[candidate]
700
                except KeyError:
701
                    # rare case: we deleted candidate in a previous iteration
702
                    # through this for loop, because it was determined to be
703
                    # a descendant of another candidate.
704
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
705
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
706
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
707
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
708
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
709
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
710
            # process found nodes
711
            new_common = set()
712
            for ancestor in ancestors:
713
                if ancestor in candidate_heads:
714
                    candidate_heads.remove(ancestor)
715
                    del searchers[ancestor]
716
                    if ancestor in active_searchers:
717
                        del active_searchers[ancestor]
718
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
719
                if ancestor in common_walker.seen:
720
                    # some searcher has encountered our known common nodes:
721
                    # just stop it
722
                    ancestor_set = set([ancestor])
723
                    for searcher in searchers.itervalues():
724
                        searcher.stop_searching_any(ancestor_set)
725
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
726
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
727
                    for searcher in searchers.itervalues():
728
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
729
                            break
730
                    else:
2921.3.4 by Robert Collins
Review feedback.
731
                        # The final active searcher has just reached this node,
732
                        # making it be known as a descendant of all candidates,
733
                        # so we can stop searching it, and any seen ancestors
734
                        new_common.add(ancestor)
735
                        for searcher in searchers.itervalues():
736
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
737
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
738
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
739
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
740
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
741
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
742
    def find_unique_lca(self, left_revision, right_revision,
743
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
744
        """Find a unique LCA.
745
746
        Find lowest common ancestors.  If there is no unique  common
747
        ancestor, find the lowest common ancestors of those ancestors.
748
749
        Iteration stops when a unique lowest common ancestor is found.
750
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
751
752
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
753
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
754
755
        :param count_steps: If True, the return value will be a tuple of
756
            (unique_lca, steps) where steps is the number of times that
757
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
758
        """
759
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
760
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
761
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
762
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
763
            lca = self.find_lca(*revisions)
764
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
765
                result = lca.pop()
766
                if count_steps:
767
                    return result, steps
768
                else:
769
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
770
            if len(lca) == 0:
771
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
772
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
773
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
774
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
775
        """Iterate the ancestry of this revision.
776
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
777
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
778
        :return: Yield tuples mapping a revision_id to its parents for the
779
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
780
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
781
            with no parents will have NULL_REVISION as their only parent. (As
782
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
783
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
784
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
785
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
786
        processed = set()
787
        while pending:
788
            processed.update(pending)
789
            next_map = self.get_parent_map(pending)
790
            next_pending = set()
791
            for item in next_map.iteritems():
792
                yield item
793
                next_pending.update(p for p in item[1] if p not in processed)
794
            ghosts = pending.difference(next_map)
795
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
796
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
797
            pending = next_pending
798
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
799
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
800
        """Iterate through the input revisions in topological order.
801
802
        This sorting only ensures that parents come before their children.
803
        An ancestor may sort after a descendant if the relationship is not
804
        visible in the supplied list of revisions.
805
        """
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
806
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
807
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
808
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
809
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
810
        """Determine whether a revision is an ancestor of another.
811
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
812
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
813
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
814
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
815
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
816
        return set([candidate_descendant]) == self.heads(
817
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
818
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
819
    def _search_for_extra_common(self, common, searchers):
820
        """Make sure that unique nodes are genuinely unique.
821
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
822
        After _find_border_ancestors, all nodes marked "common" are indeed
823
        common. Some of the nodes considered unique are not, due to history
824
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
825
826
        We know that we have searched enough when all common search tips are
827
        descended from all unique (uncommon) nodes because we know that a node
828
        cannot be an ancestor of its own ancestor.
829
830
        :param common: A set of common nodes
831
        :param searchers: The searchers returned from _find_border_ancestors
832
        :return: None
833
        """
834
        # Basic algorithm...
835
        #   A) The passed in searchers should all be on the same tips, thus
836
        #      they should be considered the "common" searchers.
837
        #   B) We find the difference between the searchers, these are the
838
        #      "unique" nodes for each side.
839
        #   C) We do a quick culling so that we only start searching from the
840
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
841
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
842
        #   D) We start searching for ancestors common to all unique nodes.
843
        #   E) We have the common searchers stop searching any ancestors of
844
        #      nodes found by (D)
845
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
846
847
        # TODO: We need a way to remove unique_searchers when they overlap with
848
        #       other unique searchers.
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
849
        if len(searchers) != 2:
850
            raise NotImplementedError(
851
                "Algorithm not yet implemented for > 2 searchers")
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
852
        common_searchers = searchers
853
        left_searcher = searchers[0]
854
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
855
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
856
        if not unique: # No unique nodes, nothing to do
857
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
858
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
859
        unique = self._remove_simple_descendants(unique,
860
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
861
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
862
863
        unique_searchers = []
864
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
865
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
866
                parent_searcher = left_searcher
867
            else:
868
                parent_searcher = right_searcher
869
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
870
            if not revs_to_search: # XXX: This shouldn't be possible
871
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
872
            searcher = self._make_breadth_first_searcher(revs_to_search)
873
            # We don't care about the starting nodes.
874
            searcher.step()
875
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
876
3377.3.16 by John Arbash Meinel
small cleanups
877
        # possible todo: aggregate the common searchers into a single common
878
        #   searcher, just make sure that we include the nodes into the .seen
879
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
880
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
881
        ancestor_all_unique = None
882
        for searcher in unique_searchers:
883
            if ancestor_all_unique is None:
884
                ancestor_all_unique = set(searcher.seen)
885
            else:
886
                ancestor_all_unique = ancestor_all_unique.intersection(
887
                                            searcher.seen)
888
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
889
        trace.mutter('Started %s unique searchers for %s unique revisions',
890
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
891
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
892
        while True: # If we have no more nodes we have nothing to do
893
            newly_seen_common = set()
894
            for searcher in common_searchers:
895
                newly_seen_common.update(searcher.step())
896
            newly_seen_unique = set()
897
            for searcher in unique_searchers:
898
                newly_seen_unique.update(searcher.step())
899
            new_common_unique = set()
900
            for revision in newly_seen_unique:
901
                for searcher in unique_searchers:
902
                    if revision not in searcher.seen:
903
                        break
904
                else:
905
                    # This is a border because it is a first common that we see
906
                    # after walking for a while.
907
                    new_common_unique.add(revision)
908
            if newly_seen_common:
909
                # These are nodes descended from one of the 'common' searchers.
910
                # Make sure all searchers are on the same page
911
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
912
                    newly_seen_common.update(
913
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
914
                # We start searching the whole ancestry. It is a bit wasteful,
915
                # though. We really just want to mark all of these nodes as
916
                # 'seen' and then start just the tips. However, it requires a
917
                # get_parent_map() call to figure out the tips anyway, and all
918
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
919
                for searcher in common_searchers:
920
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
921
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
922
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
923
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
924
                stop_searching_common = ancestor_all_unique.intersection(
925
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
926
                if stop_searching_common:
927
                    for searcher in common_searchers:
928
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
929
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
930
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
931
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
932
                    new_common_unique.update(
933
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
934
                # Since these are common, we can grab another set of ancestors
935
                # that we have seen
936
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
937
                    new_common_unique.update(
938
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
939
940
                # We can tell all of the unique searchers to start at these
941
                # nodes, and tell all of the common searchers to *stop*
942
                # searching these nodes
943
                for searcher in unique_searchers:
944
                    searcher.start_searching(new_common_unique)
945
                for searcher in common_searchers:
946
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
947
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
948
3377.3.20 by John Arbash Meinel
comment cleanups.
949
                # Filter out searchers that don't actually search different
950
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
951
                next_unique_searchers = []
952
                unique_search_sets = set()
953
                for searcher in unique_searchers:
954
                    will_search_set = frozenset(searcher._next_query)
955
                    if will_search_set not in unique_search_sets:
956
                        # This searcher is searching a unique set of nodes, let it
957
                        unique_search_sets.add(will_search_set)
958
                        next_unique_searchers.append(searcher)
959
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
960
            for searcher in common_searchers:
961
                if searcher._next_query:
962
                    break
963
            else:
964
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
965
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
966
967
    def _remove_simple_descendants(self, revisions, parent_map):
968
        """remove revisions which are children of other ones in the set
969
970
        This doesn't do any graph searching, it just checks the immediate
971
        parent_map to find if there are any children which can be removed.
972
973
        :param revisions: A set of revision_ids
974
        :return: A set of revision_ids with the children removed
975
        """
976
        simple_ancestors = revisions.copy()
977
        # TODO: jam 20071214 we *could* restrict it to searching only the
978
        #       parent_map of revisions already present in 'revisions', but
979
        #       considering the general use case, I think this is actually
980
        #       better.
981
982
        # This is the same as the following loop. I don't know that it is any
983
        # faster.
984
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
985
        ##     if p_ids is not None and revisions.intersection(p_ids))
986
        ## return simple_ancestors
987
988
        # Yet Another Way, invert the parent map (which can be cached)
989
        ## descendants = {}
990
        ## for revision_id, parent_ids in parent_map.iteritems():
991
        ##   for p_id in parent_ids:
992
        ##       descendants.setdefault(p_id, []).append(revision_id)
993
        ## for revision in revisions.intersection(descendants):
994
        ##   simple_ancestors.difference_update(descendants[revision])
995
        ## return simple_ancestors
996
        for revision, parent_ids in parent_map.iteritems():
997
            if parent_ids is None:
998
                continue
999
            for parent_id in parent_ids:
1000
                if parent_id in revisions:
1001
                    # This node has a parent present in the set, so we can
1002
                    # remove it
1003
                    simple_ancestors.discard(revision)
1004
                    break
1005
        return simple_ancestors
1006
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1007
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1008
class HeadsCache(object):
1009
    """A cache of results for graph heads calls."""
1010
1011
    def __init__(self, graph):
1012
        self.graph = graph
1013
        self._heads = {}
1014
1015
    def heads(self, keys):
1016
        """Return the heads of keys.
1017
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
1018
        This matches the API of Graph.heads(), specifically the return value is
1019
        a set which can be mutated, and ordering of the input is not preserved
1020
        in the output.
1021
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1022
        :see also: Graph.heads.
1023
        :param keys: The keys to calculate heads for.
1024
        :return: A set containing the heads, which may be mutated without
1025
            affecting future lookups.
1026
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
1027
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1028
        try:
1029
            return set(self._heads[keys])
1030
        except KeyError:
1031
            heads = self.graph.heads(keys)
1032
            self._heads[keys] = heads
1033
            return set(heads)
1034
1035
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1036
class FrozenHeadsCache(object):
1037
    """Cache heads() calls, assuming the caller won't modify them."""
1038
1039
    def __init__(self, graph):
1040
        self.graph = graph
1041
        self._heads = {}
1042
1043
    def heads(self, keys):
1044
        """Return the heads of keys.
1045
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1046
        Similar to Graph.heads(). The main difference is that the return value
1047
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1048
1049
        :see also: Graph.heads.
1050
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1051
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1052
        """
1053
        keys = frozenset(keys)
1054
        try:
1055
            return self._heads[keys]
1056
        except KeyError:
1057
            heads = frozenset(self.graph.heads(keys))
1058
            self._heads[keys] = heads
1059
            return heads
1060
1061
    def cache(self, keys, heads):
1062
        """Store a known value."""
1063
        self._heads[frozenset(keys)] = frozenset(heads)
1064
1065
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1066
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
1067
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1068
1069
    This class implements the iterator protocol, but additionally
1070
    1. provides a set of seen ancestors, and
1071
    2. allows some ancestries to be unsearched, via stop_searching_any
1072
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1073
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1074
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1075
        self._iterations = 0
1076
        self._next_query = set(revisions)
1077
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1078
        self._started_keys = set(self._next_query)
1079
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1080
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
1081
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1082
        self._current_present = set()
1083
        self._current_ghosts = set()
1084
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1085
1086
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1087
        if self._iterations:
1088
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1089
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1090
            prefix = "starting"
1091
        search = '%s=%r' % (prefix, list(self._next_query))
1092
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
1093
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1094
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1095
    def get_result(self):
1096
        """Get a SearchResult for the current state of this searcher.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1097
        
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1098
        :return: A SearchResult for this search so far. The SearchResult is
1099
            static - the search can be advanced and the search result will not
1100
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1101
        """
1102
        if self._returning == 'next':
1103
            # We have to know the current nodes children to be able to list the
1104
            # exclude keys for them. However, while we could have a second
1105
            # look-ahead result buffer and shuffle things around, this method
1106
            # is typically only called once per search - when memoising the
3211.5.1 by Robert Collins
Change the smart server get_parents method to take a graph search to exclude already recieved parents from. This prevents history shortcuts causing huge numbers of duplicates.
1107
            # results of the search. 
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1108
            found, ghosts, next, parents = self._do_query(self._next_query)
1109
            # pretend we didn't query: perhaps we should tweak _do_query to be
1110
            # entirely stateless?
1111
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1112
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1113
        else:
1114
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
1115
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1116
        included_keys = self.seen.difference(excludes)
1117
        return SearchResult(self._started_keys, excludes, len(included_keys),
1118
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1119
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1120
    def step(self):
1121
        try:
1122
            return self.next()
1123
        except StopIteration:
1124
            return ()
1125
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1126
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1127
        """Return the next ancestors of this revision.
1128
2490.2.12 by Aaron Bentley
Improve documentation
1129
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1130
        traversal.  No ancestor will be returned more than once. Ancestors are
1131
        returned before their parentage is queried, so ghosts and missing
1132
        revisions (including the start revisions) are included in the result.
1133
        This can save a round trip in LCA style calculation by allowing
1134
        convergence to be detected without reading the data for the revision
1135
        the convergence occurs on.
1136
1137
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1138
        """
3177.3.3 by Robert Collins
Review feedback.
1139
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1140
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
1141
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1142
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1143
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1144
            self._advance()
1145
        if len(self._next_query) == 0:
1146
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1147
        # We have seen what we're querying at this point as we are returning
1148
        # the query, not the results.
1149
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1150
        return self._next_query
1151
1152
    def next_with_ghosts(self):
1153
        """Return the next found ancestors, with ghosts split out.
1154
        
1155
        Ancestors are returned in the order they are seen in a breadth-first
1156
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
1157
        returned only after asking for their parents, which allows us to detect
1158
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1159
1160
        :return: A tuple with (present ancestors, ghost ancestors) sets.
1161
        """
3177.3.3 by Robert Collins
Review feedback.
1162
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1163
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
1164
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1165
            self._advance()
1166
        if len(self._next_query) == 0:
1167
            raise StopIteration()
1168
        self._advance()
1169
        return self._current_present, self._current_ghosts
1170
1171
    def _advance(self):
1172
        """Advance the search.
1173
1174
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1175
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1176
        """
1177
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1178
        found, ghosts, next, parents = self._do_query(self._next_query)
1179
        self._current_present = found
1180
        self._current_ghosts = ghosts
1181
        self._next_query = next
1182
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1183
        # ghosts are implicit stop points, otherwise the search cannot be
1184
        # repeated when ghosts are filled.
1185
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1186
1187
    def _do_query(self, revisions):
1188
        """Query for revisions.
1189
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1190
        Adds revisions to the seen set.
1191
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1192
        :param revisions: Revisions to query.
1193
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1194
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1195
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1196
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1197
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1198
        # revisions may contain nodes that point to other nodes in revisions:
1199
        # we want to filter them out.
1200
        self.seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1201
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1202
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1203
        for rev_id, parents in parent_map.iteritems():
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1204
            new_found_parents = [p for p in parents if p not in self.seen]
1205
            if new_found_parents:
1206
                # Calling set.update() with an empty generator is actually
1207
                # rather expensive.
1208
                parents_of_found.update(new_found_parents)
1209
        ghost_revisions = revisions - found_revisions
1210
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1211
2490.2.8 by Aaron Bentley
fix iteration stuff
1212
    def __iter__(self):
1213
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1214
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1215
    def find_seen_ancestors(self, revisions):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1216
        """Find ancestors of these revisions that have already been seen.
1217
        
1218
        This function generally makes the assumption that querying for the
1219
        parents of a node that has already been queried is reasonably cheap.
1220
        (eg, not a round trip to a remote host).
1221
        """
1222
        # TODO: Often we might ask one searcher for its seen ancestors, and
1223
        #       then ask another searcher the same question. This can result in
1224
        #       searching the same revisions repeatedly if the two searchers
1225
        #       have a lot of overlap.
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1226
        all_seen = self.seen
1227
        pending = set(revisions).intersection(all_seen)
1228
        seen_ancestors = set(pending)
1229
1230
        if self._returning == 'next':
1231
            # self.seen contains what nodes have been returned, not what nodes
1232
            # have been queried. We don't want to probe for nodes that haven't
1233
            # been searched yet.
1234
            not_searched_yet = self._next_query
1235
        else:
1236
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1237
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1238
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1239
        while pending:
1240
            parent_map = get_parent_map(pending)
1241
            all_parents = []
1242
            # We don't care if it is a ghost, since it can't be seen if it is
1243
            # a ghost
1244
            for parent_ids in parent_map.itervalues():
1245
                all_parents.extend(parent_ids)
1246
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1247
            seen_ancestors.update(next_pending)
1248
            next_pending.difference_update(not_searched_yet)
1249
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1250
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1251
        return seen_ancestors
1252
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1253
    def stop_searching_any(self, revisions):
1254
        """
1255
        Remove any of the specified revisions from the search list.
1256
1257
        None of the specified revisions are required to be present in the
2490.2.12 by Aaron Bentley
Improve documentation
1258
        search list.  In this case, the call is a no-op.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1259
        """
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1260
        # TODO: does this help performance?
1261
        # if not revisions:
1262
        #     return set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1263
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1264
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1265
            stopped = self._next_query.intersection(revisions)
1266
            self._next_query = self._next_query.difference(revisions)
1267
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1268
            stopped_present = self._current_present.intersection(revisions)
1269
            stopped = stopped_present.union(
1270
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1271
            self._current_present.difference_update(stopped)
1272
            self._current_ghosts.difference_update(stopped)
1273
            # stopping 'x' should stop returning parents of 'x', but 
1274
            # not if 'y' always references those same parents
1275
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1276
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1277
                for parent_id in self._current_parents[rev]:
1278
                    if parent_id not in stop_rev_references:
1279
                        stop_rev_references[parent_id] = 0
1280
                    stop_rev_references[parent_id] += 1
1281
            # if only the stopped revisions reference it, the ref count will be
1282
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1283
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1284
                for parent_id in parents:
1285
                    try:
1286
                        stop_rev_references[parent_id] -= 1
1287
                    except KeyError:
1288
                        pass
1289
            stop_parents = set()
1290
            for rev_id, refs in stop_rev_references.iteritems():
1291
                if refs == 0:
1292
                    stop_parents.add(rev_id)
1293
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1294
        self._stopped_keys.update(stopped)
2490.2.25 by Aaron Bentley
Update from review
1295
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1296
1297
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1298
        """Add revisions to the search.
1299
1300
        The parents of revisions will be returned from the next call to next()
1301
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1302
        next* call then the return value is the result of looking up the
1303
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1304
        """
1305
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1306
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1307
        new_revisions = revisions.difference(self.seen)
3177.3.3 by Robert Collins
Review feedback.
1308
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1309
            self._next_query.update(new_revisions)
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1310
            self.seen.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1311
        else:
1312
            # perform a query on revisions
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1313
            revs, ghosts, query, parents = self._do_query(revisions)
1314
            self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1315
            self._current_present.update(revs)
1316
            self._current_ghosts.update(ghosts)
1317
            self._next_query.update(query)
1318
            self._current_parents.update(parents)
1319
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1320
1321
1322
class SearchResult(object):
1323
    """The result of a breadth first search.
1324
1325
    A SearchResult provides the ability to reconstruct the search or access a
1326
    set of the keys the search found.
1327
    """
1328
1329
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1330
        """Create a SearchResult.
1331
1332
        :param start_keys: The keys the search started at.
1333
        :param exclude_keys: The keys the search excludes.
1334
        :param key_count: The total number of keys (from start to but not
1335
            including exclude).
1336
        :param keys: The keys the search found. Note that in future we may get
1337
            a SearchResult from a smart server, in which case the keys list is
1338
            not necessarily immediately available.
1339
        """
1340
        self._recipe = (start_keys, exclude_keys, key_count)
1341
        self._keys = frozenset(keys)
1342
1343
    def get_recipe(self):
1344
        """Return a recipe that can be used to replay this search.
1345
        
1346
        The recipe allows reconstruction of the same results at a later date
1347
        without knowing all the found keys. The essential elements are a list
1348
        of keys to start and and to stop at. In order to give reproducible
1349
        results when ghosts are encountered by a search they are automatically
1350
        added to the exclude list (or else ghost filling may alter the
1351
        results).
1352
1353
        :return: A tuple (start_keys_set, exclude_keys_set, revision_count). To
1354
            recreate the results of this search, create a breadth first
1355
            searcher on the same graph starting at start_keys. Then call next()
1356
            (or next_with_ghosts()) repeatedly, and on every result, call
1357
            stop_searching_any on any keys from the exclude_keys set. The
1358
            revision_count value acts as a trivial cross-check - the found
1359
            revisions of the new search should have as many elements as
1360
            revision_count. If it does not, then additional revisions have been
1361
            ghosted since the search was executed the first time and the second
1362
            time.
1363
        """
1364
        return self._recipe
1365
1366
    def get_keys(self):
1367
        """Return the keys found in this search.
1368
1369
        :return: A set of keys.
1370
        """
1371
        return self._keys
1372