/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
17
from bzrlib import (
18
    errors,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
19
    revision,
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
20
    symbol_versioning,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
21
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
22
    tsort,
23
    )
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
24
from bzrlib.deprecated_graph import (node_distances, select_farthest)
2490.2.1 by Aaron Bentley
Start work on GraphWalker
25
2490.2.25 by Aaron Bentley
Update from review
26
# DIAGRAM of terminology
27
#       A
28
#       /\
29
#      B  C
30
#      |  |\
31
#      D  E F
32
#      |\/| |
33
#      |/\|/
34
#      G  H
35
#
36
# In this diagram, relative to G and H:
37
# A, B, C, D, E are common ancestors.
38
# C, D and E are border ancestors, because each has a non-common descendant.
39
# D and E are least common ancestors because none of their descendants are
40
# common ancestors.
41
# C is not a least common ancestor because its descendant, E, is a common
42
# ancestor.
43
#
44
# The find_unique_lca algorithm will pick A in two steps:
45
# 1. find_lca('G', 'H') => ['D', 'E']
46
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
47
48
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
49
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
50
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
51
52
    def __init__(self, ancestry):
53
        self.ancestry = ancestry
54
55
    def __repr__(self):
56
        return 'DictParentsProvider(%r)' % self.ancestry
57
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
58
    def get_parent_map(self, keys):
59
        """See _StackedParentsProvider.get_parent_map"""
60
        ancestry = self.ancestry
61
        return dict((k, ancestry[k]) for k in keys if k in ancestry)
62
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
63
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
64
class _StackedParentsProvider(object):
65
66
    def __init__(self, parent_providers):
67
        self._parent_providers = parent_providers
68
2490.2.28 by Aaron Bentley
Fix handling of null revision
69
    def __repr__(self):
70
        return "_StackedParentsProvider(%r)" % self._parent_providers
71
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
72
    def get_parent_map(self, keys):
73
        """Get a mapping of keys => parents
74
75
        A dictionary is returned with an entry for each key present in this
76
        source. If this source doesn't have information about a key, it should
77
        not include an entry.
78
79
        [NULL_REVISION] is used as the parent of the first user-committed
80
        revision.  Its parent list is empty.
81
82
        :param keys: An iterable returning keys to check (eg revision_ids)
83
        :return: A dictionary mapping each key to its parents
84
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
85
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
86
        remaining = set(keys)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
87
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
88
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
89
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
90
            remaining.difference_update(new_found)
91
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
92
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
93
        return found
94
95
96
class CachingParentsProvider(object):
97
    """A parents provider which will cache the revision => parents in a dict.
98
99
    This is useful for providers that have an expensive lookup.
100
    """
101
102
    def __init__(self, parent_provider):
103
        self._real_provider = parent_provider
104
        # Theoretically we could use an LRUCache here
105
        self._cache = {}
106
107
    def __repr__(self):
108
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
109
110
    def get_parent_map(self, keys):
111
        """See _StackedParentsProvider.get_parent_map"""
112
        needed = set()
113
        # If the _real_provider doesn't have a key, we cache a value of None,
114
        # which we then later use to realize we cannot provide a value for that
115
        # key.
116
        parent_map = {}
117
        cache = self._cache
118
        for key in keys:
119
            if key in cache:
120
                value = cache[key]
121
                if value is not None:
122
                    parent_map[key] = value
123
            else:
124
                needed.add(key)
125
126
        if needed:
127
            new_parents = self._real_provider.get_parent_map(needed)
128
            cache.update(new_parents)
129
            parent_map.update(new_parents)
130
            needed.difference_update(new_parents)
131
            cache.update(dict.fromkeys(needed, None))
132
        return parent_map
133
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
134
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
135
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
136
    """Provide incremental access to revision graphs.
137
138
    This is the generic implementation; it is intended to be subclassed to
139
    specialize it for other repository types.
140
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
141
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
142
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
143
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
144
145
        This should not normally be invoked directly, because there may be
146
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
147
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
148
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
149
        :param parents_provider: An object providing a get_parent_map call
150
            conforming to the behavior of
151
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
152
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
153
        if getattr(parents_provider, 'get_parents', None) is not None:
154
            self.get_parents = parents_provider.get_parents
155
        if getattr(parents_provider, 'get_parent_map', None) is not None:
156
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
157
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
158
159
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
160
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
161
3377.3.28 by John Arbash Meinel
Try using _find_any_seen_ancestors,
162
    def _find_any_seen_ancestors(self, revisions, searchers):
163
        """Find any revisions that are seen by any of the searchers."""
164
        # This will actually return more nodes than just aggregating
165
        # find_seen_ancestors() across the searchers, because it can bridge
166
        # 1-node gaps inline.
167
168
        # seen contains what nodes have been returned, not what nodes
169
        # have been queried. We don't want to probe for nodes that haven't
170
        # been searched yet.
171
        all_seen = set()
172
        not_searched_yet = set()
173
        for searcher in searchers:
174
            all_seen.update(searcher.seen)
175
176
            not_searched = set(searcher._next_query)
177
            # Have these nodes been searched in any other searcher?
178
            for other_searcher in searchers:
179
                if other_searcher is searcher:
180
                    continue
181
                # This other searcher claims to have seen these nodes
182
                maybe_searched = not_searched.intersection(other_searcher.seen)
183
                searched = maybe_searched.difference(other_searcher._next_query)
184
                not_searched.difference_update(searched)
185
186
            # Now we know the real ones that haven't been searched
187
            not_searched_yet.update(not_searched)
188
189
        pending = set(revisions).intersection(all_seen)
190
        seen_ancestors = set(pending)
191
192
        pending.difference_update(not_searched_yet)
193
        get_parent_map = self._parents_provider.get_parent_map
194
        while pending:
195
            parent_map = get_parent_map(pending)
196
            all_parents = []
197
            # We don't care if it is a ghost, since it can't be seen if it is
198
            # a ghost
199
            for parent_ids in parent_map.itervalues():
200
                all_parents.extend(parent_ids)
201
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
202
            seen_ancestors.update(next_pending)
203
            next_pending.difference_update(not_searched_yet)
204
            pending = next_pending
205
206
        return seen_ancestors
207
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
208
    def find_lca(self, *revisions):
209
        """Determine the lowest common ancestors of the provided revisions
210
211
        A lowest common ancestor is a common ancestor none of whose
212
        descendants are common ancestors.  In graphs, unlike trees, there may
213
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
214
215
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
216
        and phase 2 filters border ancestors to determine lowest common
217
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
218
219
        In phase 1, border ancestors are identified, using a breadth-first
220
        search starting at the bottom of the graph.  Searches are stopped
221
        whenever a node or one of its descendants is determined to be common
222
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
223
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
224
        common ancestors.  This is done by searching the ancestries of each
225
        border ancestor.
226
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
227
        Phase 2 is perfomed on the principle that a border ancestor that is
228
        not an ancestor of any other border ancestor is a least common
229
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
230
231
        Searches are stopped when they find a node that is determined to be a
232
        common ancestor of all border ancestors, because this shows that it
233
        cannot be a descendant of any border ancestor.
234
235
        The scaling of this operation should be proportional to
236
        1. The number of uncommon ancestors
237
        2. The number of border ancestors
238
        3. The length of the shortest path between a border ancestor and an
239
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
240
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
241
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
242
        # We may have common ancestors that can be reached from each other.
243
        # - ask for the heads of them to filter it down to only ones that
244
        # cannot be reached from each other - phase 2.
245
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
246
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
247
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
248
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
249
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
250
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
251
        self._search_for_extra_common(common, searchers)
252
        left = searchers[0].seen
253
        right = searchers[1].seen
254
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
255
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
256
    def find_unique_ancestors(self, unique_revision, common_revisions):
257
        """Find the unique ancestors for a revision versus others.
258
259
        This returns the ancestry of unique_revision, excluding all revisions
260
        in the ancestry of common_revisions. If unique_revision is in the
261
        ancestry, then the empty set will be returned.
262
263
        :param unique_revision: The revision_id whose ancestry we are
264
            interested in.
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
265
            XXX: Would this API be better if we allowed multiple revisions on
266
                 to be searched here?
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
267
        :param common_revisions: Revision_ids of ancestries to exclude.
268
        :return: A set of revisions in the ancestry of unique_revision
269
        """
270
        if unique_revision in common_revisions:
271
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
272
273
        # Algorithm description
274
        # 1) Walk backwards from the unique node and all common nodes.
275
        # 2) When a node is seen by both sides, stop searching it in the unique
276
        #    walker, include it in the common walker.
277
        # 3) Stop searching when there are no nodes left for the unique walker.
278
        #    At this point, you have a maximal set of unique nodes. Some of
279
        #    them may actually be common, and you haven't reached them yet.
280
        # 4) Start new searchers for the unique nodes, seeded with the
281
        #    information you have so far.
282
        # 5) Continue searching, stopping the common searches when the search
283
        #    tip is an ancestor of all unique nodes.
284
        # 6) Search is done when all common searchers have completed.
285
286
        unique_searcher = self._make_breadth_first_searcher([unique_revision])
3377.3.27 by John Arbash Meinel
some simple updates
287
        # we know that unique_revision isn't in common_revisions
288
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
289
        common_searcher = self._make_breadth_first_searcher(common_revisions)
290
3377.3.27 by John Arbash Meinel
some simple updates
291
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
292
            next_unique_nodes = set(unique_searcher.step())
293
            next_common_nodes = set(common_searcher.step())
294
295
            # Check if either searcher encounters new nodes seen by the other
296
            # side.
297
            unique_are_common_nodes = next_unique_nodes.intersection(
298
                common_searcher.seen)
299
            unique_are_common_nodes.update(
300
                next_common_nodes.intersection(unique_searcher.seen))
301
            if unique_are_common_nodes:
3377.3.28 by John Arbash Meinel
Try using _find_any_seen_ancestors,
302
                ancestors = self._find_any_seen_ancestors(unique_are_common_nodes,
303
                    [unique_searcher, common_searcher])
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
304
                unique_searcher.stop_searching_any(ancestors)
305
                common_searcher.start_searching(ancestors)
306
307
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
308
        if not unique_nodes:
309
            return unique_nodes
310
        unique_tips = self._remove_simple_descendants(unique_nodes,
311
                        self.get_parent_map(unique_nodes))
312
313
        unique_searchers = []
314
        for tip in unique_tips:
315
            revs_to_search = unique_searcher.find_seen_ancestors([tip])
316
            searcher = self._make_breadth_first_searcher(revs_to_search)
317
            # We don't care about the starting nodes.
318
            searcher.step()
319
            unique_searchers.append(searcher)
320
321
        ancestor_all_unique = None
322
        for searcher in unique_searchers:
323
            if ancestor_all_unique is None:
324
                ancestor_all_unique = set(searcher.seen)
325
            else:
326
                ancestor_all_unique = ancestor_all_unique.intersection(
327
                                            searcher.seen)
328
329
        # Stop any search tips that are already known as ancestors of the
330
        # unique nodes
3377.3.28 by John Arbash Meinel
Try using _find_any_seen_ancestors,
331
        common_searcher.stop_searching_any(
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
332
            common_searcher.find_seen_ancestors(ancestor_all_unique))
333
334
        # While we still have common nodes to search
335
        while common_searcher._next_query:
336
            newly_seen_common = set(common_searcher.step())
337
            newly_seen_unique = set()
338
            for searcher in unique_searchers:
339
                newly_seen_unique.update(searcher.step())
340
            # These nodes are common ancestors of all unique nodes
341
            unique_are_common_nodes = newly_seen_unique.copy()
342
            for searcher in unique_searchers:
343
                unique_are_common_nodes = unique_are_common_nodes.intersection(
344
                                            searcher.seen)
345
            ancestor_all_unique.update(unique_are_common_nodes)
346
            if newly_seen_common:
347
                # If a 'common' node is an ancestor of all unique searchers, we
348
                # can stop searching it.
349
                common_searcher.stop_searching_any(
350
                    ancestor_all_unique.intersection(newly_seen_common))
351
            if unique_are_common_nodes:
352
                # We have new common-to-all-unique-searchers nodes
3377.3.28 by John Arbash Meinel
Try using _find_any_seen_ancestors,
353
                # Which also means we can check in the common_searcher
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
354
                unique_are_common_nodes.update(
3377.3.28 by John Arbash Meinel
Try using _find_any_seen_ancestors,
355
                    self._find_any_seen_ancestors(unique_are_common_nodes,
356
                        [common_searcher] + unique_searchers))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
357
358
                # We can tell all of the unique searchers to start at these
359
                # nodes, and tell all of the common searchers to *stop*
360
                # searching these nodes
361
                for searcher in unique_searchers:
362
                    searcher.start_searching(unique_are_common_nodes)
363
                common_searcher.stop_searching_any(unique_are_common_nodes)
364
365
                # Filter out searchers that don't actually search different
366
                # nodes. We already have the ancestry intersection for them
367
                next_unique_searchers = []
368
                unique_search_sets = set()
369
                for searcher in unique_searchers:
370
                    will_search_set = frozenset(searcher._next_query)
371
                    if will_search_set not in unique_search_sets:
372
                        # This searcher is searching a unique set of nodes, let it
373
                        unique_search_sets.add(will_search_set)
374
                        next_unique_searchers.append(searcher)
375
                unique_searchers = next_unique_searchers
376
        return unique_nodes.difference(common_searcher.seen)
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
377
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
378
    @symbol_versioning.deprecated_method(symbol_versioning.one_one)
379
    def get_parents(self, revisions):
380
        """Find revision ids of the parents of a list of revisions
381
382
        A list is returned of the same length as the input.  Each entry
383
        is a list of parent ids for the corresponding input revision.
384
385
        [NULL_REVISION] is used as the parent of the first user-committed
386
        revision.  Its parent list is empty.
387
388
        If the revision is not present (i.e. a ghost), None is used in place
389
        of the list of parents.
390
391
        Deprecated in bzr 1.2 - please see get_parent_map.
392
        """
393
        parents = self.get_parent_map(revisions)
3377.3.5 by John Arbash Meinel
Fix a latent bug in Graph.get_parents()
394
        return [parents.get(r, None) for r in revisions]
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
395
396
    def get_parent_map(self, revisions):
397
        """Get a map of key:parent_list for revisions.
398
399
        This implementation delegates to get_parents, for old parent_providers
400
        that do not supply get_parent_map.
401
        """
402
        result = {}
403
        for rev, parents in self.get_parents(revisions):
404
            if parents is not None:
405
                result[rev] = parents
406
        return result
407
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
408
    def _make_breadth_first_searcher(self, revisions):
409
        return _BreadthFirstSearcher(revisions, self)
410
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
411
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
412
        """Find common ancestors with at least one uncommon descendant.
413
414
        Border ancestors are identified using a breadth-first
415
        search starting at the bottom of the graph.  Searches are stopped
416
        whenever a node or one of its descendants is determined to be common.
417
418
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
419
420
        As well as the border ancestors, a set of seen common ancestors and a
421
        list of sets of seen ancestors for each input revision is returned.
422
        This allows calculation of graph difference from the results of this
423
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
424
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
425
        if None in revisions:
426
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
427
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
428
        searchers = [self._make_breadth_first_searcher([r])
429
                     for r in revisions]
430
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
431
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
432
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
433
        while True:
434
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
435
            for searcher in searchers:
436
                new_ancestors = searcher.step()
437
                if new_ancestors:
438
                    newly_seen.update(new_ancestors)
439
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
440
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
441
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
442
                    # Not a border ancestor because it was seen as common
443
                    # already
444
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
445
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
446
                for searcher in searchers:
447
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
448
                        break
449
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
450
                    # This is a border because it is a first common that we see
451
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
452
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
453
                    new_common.add(revision)
454
            if new_common:
455
                for searcher in searchers:
456
                    new_common.update(searcher.find_seen_ancestors(new_common))
457
                for searcher in searchers:
458
                    searcher.start_searching(new_common)
459
                common_ancestors.update(new_common)
460
461
            # Figure out what the searchers will be searching next, and if
462
            # there is only 1 set being searched, then we are done searching,
463
            # since all searchers would have to be searching the same data,
464
            # thus it *must* be in common.
465
            unique_search_sets = set()
466
            for searcher in searchers:
467
                will_search_set = frozenset(searcher._next_query)
468
                if will_search_set not in unique_search_sets:
469
                    # This searcher is searching a unique set of nodes, let it
470
                    unique_search_sets.add(will_search_set)
471
472
            if len(unique_search_sets) == 1:
473
                nodes = unique_search_sets.pop()
474
                uncommon_nodes = nodes.difference(common_ancestors)
475
                assert not uncommon_nodes, ("Somehow we ended up converging"
476
                                            " without actually marking them as"
477
                                            " in common."
478
                                            "\nStart_nodes: %s"
479
                                            "\nuncommon_nodes: %s"
480
                                            % (revisions, uncommon_nodes))
481
                break
482
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
483
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
484
    def heads(self, keys):
485
        """Return the heads from amongst keys.
486
487
        This is done by searching the ancestries of each key.  Any key that is
488
        reachable from another key is not returned; all the others are.
489
490
        This operation scales with the relative depth between any two keys. If
491
        any two keys are completely disconnected all ancestry of both sides
492
        will be retrieved.
493
494
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
495
        :return: A set of the heads. Note that as a set there is no ordering
496
            information. Callers will need to filter their input to create
497
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
498
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
499
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
500
        if revision.NULL_REVISION in candidate_heads:
501
            # NULL_REVISION is only a head if it is the only entry
502
            candidate_heads.remove(revision.NULL_REVISION)
503
            if not candidate_heads:
504
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
505
        if len(candidate_heads) < 2:
506
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
507
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
508
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
509
        active_searchers = dict(searchers)
510
        # skip over the actual candidate for each searcher
511
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
512
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
513
        # The common walker finds nodes that are common to two or more of the
514
        # input keys, so that we don't access all history when a currently
515
        # uncommon search point actually meets up with something behind a
516
        # common search point. Common search points do not keep searches
517
        # active; they just allow us to make searches inactive without
518
        # accessing all history.
519
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
520
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
521
            ancestors = set()
522
            # advance searches
523
            try:
524
                common_walker.next()
525
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
526
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
527
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
528
            for candidate in active_searchers.keys():
529
                try:
530
                    searcher = active_searchers[candidate]
531
                except KeyError:
532
                    # rare case: we deleted candidate in a previous iteration
533
                    # through this for loop, because it was determined to be
534
                    # a descendant of another candidate.
535
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
536
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
537
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
538
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
539
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
540
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
541
            # process found nodes
542
            new_common = set()
543
            for ancestor in ancestors:
544
                if ancestor in candidate_heads:
545
                    candidate_heads.remove(ancestor)
546
                    del searchers[ancestor]
547
                    if ancestor in active_searchers:
548
                        del active_searchers[ancestor]
549
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
550
                if ancestor in common_walker.seen:
551
                    # some searcher has encountered our known common nodes:
552
                    # just stop it
553
                    ancestor_set = set([ancestor])
554
                    for searcher in searchers.itervalues():
555
                        searcher.stop_searching_any(ancestor_set)
556
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
557
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
558
                    for searcher in searchers.itervalues():
559
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
560
                            break
561
                    else:
2921.3.4 by Robert Collins
Review feedback.
562
                        # The final active searcher has just reached this node,
563
                        # making it be known as a descendant of all candidates,
564
                        # so we can stop searching it, and any seen ancestors
565
                        new_common.add(ancestor)
566
                        for searcher in searchers.itervalues():
567
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
568
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
569
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
570
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
571
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
572
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
573
    def find_unique_lca(self, left_revision, right_revision,
574
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
575
        """Find a unique LCA.
576
577
        Find lowest common ancestors.  If there is no unique  common
578
        ancestor, find the lowest common ancestors of those ancestors.
579
580
        Iteration stops when a unique lowest common ancestor is found.
581
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
582
583
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
584
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
585
586
        :param count_steps: If True, the return value will be a tuple of
587
            (unique_lca, steps) where steps is the number of times that
588
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
589
        """
590
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
591
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
592
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
593
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
594
            lca = self.find_lca(*revisions)
595
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
596
                result = lca.pop()
597
                if count_steps:
598
                    return result, steps
599
                else:
600
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
601
            if len(lca) == 0:
602
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
603
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
604
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
605
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
606
        """Iterate the ancestry of this revision.
607
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
608
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
609
        :return: Yield tuples mapping a revision_id to its parents for the
610
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
611
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
612
            with no parents will have NULL_REVISION as their only parent. (As
613
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
614
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
615
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
616
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
617
        processed = set()
618
        while pending:
619
            processed.update(pending)
620
            next_map = self.get_parent_map(pending)
621
            next_pending = set()
622
            for item in next_map.iteritems():
623
                yield item
624
                next_pending.update(p for p in item[1] if p not in processed)
625
            ghosts = pending.difference(next_map)
626
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
627
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
628
            pending = next_pending
629
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
630
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
631
        """Iterate through the input revisions in topological order.
632
633
        This sorting only ensures that parents come before their children.
634
        An ancestor may sort after a descendant if the relationship is not
635
        visible in the supplied list of revisions.
636
        """
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
637
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
638
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
639
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
640
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
641
        """Determine whether a revision is an ancestor of another.
642
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
643
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
644
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
645
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
646
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
647
        return set([candidate_descendant]) == self.heads(
648
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
649
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
650
    def _search_for_extra_common(self, common, searchers):
651
        """Make sure that unique nodes are genuinely unique.
652
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
653
        After _find_border_ancestors, all nodes marked "common" are indeed
654
        common. Some of the nodes considered unique are not, due to history
655
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
656
657
        We know that we have searched enough when all common search tips are
658
        descended from all unique (uncommon) nodes because we know that a node
659
        cannot be an ancestor of its own ancestor.
660
661
        :param common: A set of common nodes
662
        :param searchers: The searchers returned from _find_border_ancestors
663
        :return: None
664
        """
665
        # Basic algorithm...
666
        #   A) The passed in searchers should all be on the same tips, thus
667
        #      they should be considered the "common" searchers.
668
        #   B) We find the difference between the searchers, these are the
669
        #      "unique" nodes for each side.
670
        #   C) We do a quick culling so that we only start searching from the
671
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
672
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
673
        #   D) We start searching for ancestors common to all unique nodes.
674
        #   E) We have the common searchers stop searching any ancestors of
675
        #      nodes found by (D)
676
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
677
678
        # TODO: We need a way to remove unique_searchers when they overlap with
679
        #       other unique searchers.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
680
        assert len(searchers) == 2, (
681
            "Algorithm not yet implemented for > 2 searchers")
682
        common_searchers = searchers
683
        left_searcher = searchers[0]
684
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
685
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
686
        if not unique: # No unique nodes, nothing to do
687
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
688
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
689
        unique = self._remove_simple_descendants(unique,
690
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
691
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
692
693
        unique_searchers = []
694
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
695
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
696
                parent_searcher = left_searcher
697
            else:
698
                parent_searcher = right_searcher
699
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
700
            if not revs_to_search: # XXX: This shouldn't be possible
701
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
702
            searcher = self._make_breadth_first_searcher(revs_to_search)
703
            # We don't care about the starting nodes.
704
            searcher.step()
705
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
706
3377.3.16 by John Arbash Meinel
small cleanups
707
        # possible todo: aggregate the common searchers into a single common
708
        #   searcher, just make sure that we include the nodes into the .seen
709
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
710
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
711
        ancestor_all_unique = None
712
        for searcher in unique_searchers:
713
            if ancestor_all_unique is None:
714
                ancestor_all_unique = set(searcher.seen)
715
            else:
716
                ancestor_all_unique = ancestor_all_unique.intersection(
717
                                            searcher.seen)
718
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
719
        trace.mutter('Started %s unique searchers for %s unique revisions',
720
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
721
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
722
        while True: # If we have no more nodes we have nothing to do
723
            newly_seen_common = set()
724
            for searcher in common_searchers:
725
                newly_seen_common.update(searcher.step())
726
            newly_seen_unique = set()
727
            for searcher in unique_searchers:
728
                newly_seen_unique.update(searcher.step())
729
            new_common_unique = set()
730
            for revision in newly_seen_unique:
731
                for searcher in unique_searchers:
732
                    if revision not in searcher.seen:
733
                        break
734
                else:
735
                    # This is a border because it is a first common that we see
736
                    # after walking for a while.
737
                    new_common_unique.add(revision)
738
            if newly_seen_common:
739
                # These are nodes descended from one of the 'common' searchers.
740
                # Make sure all searchers are on the same page
741
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
742
                    newly_seen_common.update(
743
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
744
                # We start searching the whole ancestry. It is a bit wasteful,
745
                # though. We really just want to mark all of these nodes as
746
                # 'seen' and then start just the tips. However, it requires a
747
                # get_parent_map() call to figure out the tips anyway, and all
748
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
749
                for searcher in common_searchers:
750
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
751
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
752
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
753
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
754
                stop_searching_common = ancestor_all_unique.intersection(
755
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
756
                if stop_searching_common:
757
                    for searcher in common_searchers:
758
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
759
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
760
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
761
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
762
                    new_common_unique.update(
763
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
764
                # Since these are common, we can grab another set of ancestors
765
                # that we have seen
766
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
767
                    new_common_unique.update(
768
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
769
770
                # We can tell all of the unique searchers to start at these
771
                # nodes, and tell all of the common searchers to *stop*
772
                # searching these nodes
773
                for searcher in unique_searchers:
774
                    searcher.start_searching(new_common_unique)
775
                for searcher in common_searchers:
776
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
777
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
778
3377.3.20 by John Arbash Meinel
comment cleanups.
779
                # Filter out searchers that don't actually search different
780
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
781
                next_unique_searchers = []
782
                unique_search_sets = set()
783
                for searcher in unique_searchers:
784
                    will_search_set = frozenset(searcher._next_query)
785
                    if will_search_set not in unique_search_sets:
786
                        # This searcher is searching a unique set of nodes, let it
787
                        unique_search_sets.add(will_search_set)
788
                        next_unique_searchers.append(searcher)
789
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
790
            for searcher in common_searchers:
791
                if searcher._next_query:
792
                    break
793
            else:
794
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
795
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
796
797
    def _remove_simple_descendants(self, revisions, parent_map):
798
        """remove revisions which are children of other ones in the set
799
800
        This doesn't do any graph searching, it just checks the immediate
801
        parent_map to find if there are any children which can be removed.
802
803
        :param revisions: A set of revision_ids
804
        :return: A set of revision_ids with the children removed
805
        """
806
        simple_ancestors = revisions.copy()
807
        # TODO: jam 20071214 we *could* restrict it to searching only the
808
        #       parent_map of revisions already present in 'revisions', but
809
        #       considering the general use case, I think this is actually
810
        #       better.
811
812
        # This is the same as the following loop. I don't know that it is any
813
        # faster.
814
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
815
        ##     if p_ids is not None and revisions.intersection(p_ids))
816
        ## return simple_ancestors
817
818
        # Yet Another Way, invert the parent map (which can be cached)
819
        ## descendants = {}
820
        ## for revision_id, parent_ids in parent_map.iteritems():
821
        ##   for p_id in parent_ids:
822
        ##       descendants.setdefault(p_id, []).append(revision_id)
823
        ## for revision in revisions.intersection(descendants):
824
        ##   simple_ancestors.difference_update(descendants[revision])
825
        ## return simple_ancestors
826
        for revision, parent_ids in parent_map.iteritems():
827
            if parent_ids is None:
828
                continue
829
            for parent_id in parent_ids:
830
                if parent_id in revisions:
831
                    # This node has a parent present in the set, so we can
832
                    # remove it
833
                    simple_ancestors.discard(revision)
834
                    break
835
        return simple_ancestors
836
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
837
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
838
class HeadsCache(object):
839
    """A cache of results for graph heads calls."""
840
841
    def __init__(self, graph):
842
        self.graph = graph
843
        self._heads = {}
844
845
    def heads(self, keys):
846
        """Return the heads of keys.
847
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
848
        This matches the API of Graph.heads(), specifically the return value is
849
        a set which can be mutated, and ordering of the input is not preserved
850
        in the output.
851
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
852
        :see also: Graph.heads.
853
        :param keys: The keys to calculate heads for.
854
        :return: A set containing the heads, which may be mutated without
855
            affecting future lookups.
856
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
857
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
858
        try:
859
            return set(self._heads[keys])
860
        except KeyError:
861
            heads = self.graph.heads(keys)
862
            self._heads[keys] = heads
863
            return set(heads)
864
865
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
866
class FrozenHeadsCache(object):
867
    """Cache heads() calls, assuming the caller won't modify them."""
868
869
    def __init__(self, graph):
870
        self.graph = graph
871
        self._heads = {}
872
873
    def heads(self, keys):
874
        """Return the heads of keys.
875
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
876
        Similar to Graph.heads(). The main difference is that the return value
877
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
878
879
        :see also: Graph.heads.
880
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
881
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
882
        """
883
        keys = frozenset(keys)
884
        try:
885
            return self._heads[keys]
886
        except KeyError:
887
            heads = frozenset(self.graph.heads(keys))
888
            self._heads[keys] = heads
889
            return heads
890
891
    def cache(self, keys, heads):
892
        """Store a known value."""
893
        self._heads[frozenset(keys)] = frozenset(heads)
894
895
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
896
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
897
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
898
899
    This class implements the iterator protocol, but additionally
900
    1. provides a set of seen ancestors, and
901
    2. allows some ancestries to be unsearched, via stop_searching_any
902
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
903
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
904
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
905
        self._iterations = 0
906
        self._next_query = set(revisions)
907
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
908
        self._started_keys = set(self._next_query)
909
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
910
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
911
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
912
        self._current_present = set()
913
        self._current_ghosts = set()
914
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
915
916
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
917
        if self._iterations:
918
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
919
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
920
            prefix = "starting"
921
        search = '%s=%r' % (prefix, list(self._next_query))
922
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
923
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
924
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
925
    def get_result(self):
926
        """Get a SearchResult for the current state of this searcher.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
927
        
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
928
        :return: A SearchResult for this search so far. The SearchResult is
929
            static - the search can be advanced and the search result will not
930
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
931
        """
932
        if self._returning == 'next':
933
            # We have to know the current nodes children to be able to list the
934
            # exclude keys for them. However, while we could have a second
935
            # look-ahead result buffer and shuffle things around, this method
936
            # is typically only called once per search - when memoising the
3211.5.1 by Robert Collins
Change the smart server get_parents method to take a graph search to exclude already recieved parents from. This prevents history shortcuts causing huge numbers of duplicates.
937
            # results of the search. 
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
938
            found, ghosts, next, parents = self._do_query(self._next_query)
939
            # pretend we didn't query: perhaps we should tweak _do_query to be
940
            # entirely stateless?
941
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
942
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
943
        else:
944
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
945
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
946
        included_keys = self.seen.difference(excludes)
947
        return SearchResult(self._started_keys, excludes, len(included_keys),
948
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
949
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
950
    def step(self):
951
        try:
952
            return self.next()
953
        except StopIteration:
954
            return ()
955
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
956
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
957
        """Return the next ancestors of this revision.
958
2490.2.12 by Aaron Bentley
Improve documentation
959
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
960
        traversal.  No ancestor will be returned more than once. Ancestors are
961
        returned before their parentage is queried, so ghosts and missing
962
        revisions (including the start revisions) are included in the result.
963
        This can save a round trip in LCA style calculation by allowing
964
        convergence to be detected without reading the data for the revision
965
        the convergence occurs on.
966
967
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
968
        """
3177.3.3 by Robert Collins
Review feedback.
969
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
970
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
971
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
972
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
973
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
974
            self._advance()
975
        if len(self._next_query) == 0:
976
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
977
        # We have seen what we're querying at this point as we are returning
978
        # the query, not the results.
979
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
980
        return self._next_query
981
982
    def next_with_ghosts(self):
983
        """Return the next found ancestors, with ghosts split out.
984
        
985
        Ancestors are returned in the order they are seen in a breadth-first
986
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
987
        returned only after asking for their parents, which allows us to detect
988
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
989
990
        :return: A tuple with (present ancestors, ghost ancestors) sets.
991
        """
3177.3.3 by Robert Collins
Review feedback.
992
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
993
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
994
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
995
            self._advance()
996
        if len(self._next_query) == 0:
997
            raise StopIteration()
998
        self._advance()
999
        return self._current_present, self._current_ghosts
1000
1001
    def _advance(self):
1002
        """Advance the search.
1003
1004
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1005
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1006
        """
1007
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1008
        found, ghosts, next, parents = self._do_query(self._next_query)
1009
        self._current_present = found
1010
        self._current_ghosts = ghosts
1011
        self._next_query = next
1012
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1013
        # ghosts are implicit stop points, otherwise the search cannot be
1014
        # repeated when ghosts are filled.
1015
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1016
1017
    def _do_query(self, revisions):
1018
        """Query for revisions.
1019
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1020
        Adds revisions to the seen set.
1021
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1022
        :param revisions: Revisions to query.
1023
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1024
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1025
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1026
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1027
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1028
        # revisions may contain nodes that point to other nodes in revisions:
1029
        # we want to filter them out.
1030
        self.seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1031
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1032
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1033
        for rev_id, parents in parent_map.iteritems():
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1034
            new_found_parents = [p for p in parents if p not in self.seen]
1035
            if new_found_parents:
1036
                # Calling set.update() with an empty generator is actually
1037
                # rather expensive.
1038
                parents_of_found.update(new_found_parents)
1039
        ghost_revisions = revisions - found_revisions
1040
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1041
2490.2.8 by Aaron Bentley
fix iteration stuff
1042
    def __iter__(self):
1043
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1044
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1045
    def find_seen_ancestors(self, revisions):
1046
        """Find ancestors of these revisions that have already been seen."""
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1047
        all_seen = self.seen
1048
        pending = set(revisions).intersection(all_seen)
1049
        seen_ancestors = set(pending)
1050
1051
        if self._returning == 'next':
1052
            # self.seen contains what nodes have been returned, not what nodes
1053
            # have been queried. We don't want to probe for nodes that haven't
1054
            # been searched yet.
1055
            not_searched_yet = self._next_query
1056
        else:
1057
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1058
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1059
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1060
        while pending:
1061
            parent_map = get_parent_map(pending)
1062
            all_parents = []
1063
            # We don't care if it is a ghost, since it can't be seen if it is
1064
            # a ghost
1065
            for parent_ids in parent_map.itervalues():
1066
                all_parents.extend(parent_ids)
1067
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1068
            seen_ancestors.update(next_pending)
1069
            next_pending.difference_update(not_searched_yet)
1070
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1071
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1072
        return seen_ancestors
1073
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1074
    def stop_searching_any(self, revisions):
1075
        """
1076
        Remove any of the specified revisions from the search list.
1077
1078
        None of the specified revisions are required to be present in the
2490.2.12 by Aaron Bentley
Improve documentation
1079
        search list.  In this case, the call is a no-op.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1080
        """
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1081
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1082
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1083
            stopped = self._next_query.intersection(revisions)
1084
            self._next_query = self._next_query.difference(revisions)
1085
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1086
            stopped_present = self._current_present.intersection(revisions)
1087
            stopped = stopped_present.union(
1088
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1089
            self._current_present.difference_update(stopped)
1090
            self._current_ghosts.difference_update(stopped)
1091
            # stopping 'x' should stop returning parents of 'x', but 
1092
            # not if 'y' always references those same parents
1093
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1094
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1095
                for parent_id in self._current_parents[rev]:
1096
                    if parent_id not in stop_rev_references:
1097
                        stop_rev_references[parent_id] = 0
1098
                    stop_rev_references[parent_id] += 1
1099
            # if only the stopped revisions reference it, the ref count will be
1100
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1101
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1102
                for parent_id in parents:
1103
                    try:
1104
                        stop_rev_references[parent_id] -= 1
1105
                    except KeyError:
1106
                        pass
1107
            stop_parents = set()
1108
            for rev_id, refs in stop_rev_references.iteritems():
1109
                if refs == 0:
1110
                    stop_parents.add(rev_id)
1111
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1112
        self._stopped_keys.update(stopped)
2490.2.25 by Aaron Bentley
Update from review
1113
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1114
1115
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1116
        """Add revisions to the search.
1117
1118
        The parents of revisions will be returned from the next call to next()
1119
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1120
        next* call then the return value is the result of looking up the
1121
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1122
        """
1123
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1124
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1125
        new_revisions = revisions.difference(self.seen)
1126
        revs, ghosts, query, parents = self._do_query(revisions)
1127
        self._stopped_keys.update(ghosts)
3177.3.3 by Robert Collins
Review feedback.
1128
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1129
            self._next_query.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1130
        else:
1131
            # perform a query on revisions
1132
            self._current_present.update(revs)
1133
            self._current_ghosts.update(ghosts)
1134
            self._next_query.update(query)
1135
            self._current_parents.update(parents)
1136
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1137
1138
1139
class SearchResult(object):
1140
    """The result of a breadth first search.
1141
1142
    A SearchResult provides the ability to reconstruct the search or access a
1143
    set of the keys the search found.
1144
    """
1145
1146
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1147
        """Create a SearchResult.
1148
1149
        :param start_keys: The keys the search started at.
1150
        :param exclude_keys: The keys the search excludes.
1151
        :param key_count: The total number of keys (from start to but not
1152
            including exclude).
1153
        :param keys: The keys the search found. Note that in future we may get
1154
            a SearchResult from a smart server, in which case the keys list is
1155
            not necessarily immediately available.
1156
        """
1157
        self._recipe = (start_keys, exclude_keys, key_count)
1158
        self._keys = frozenset(keys)
1159
1160
    def get_recipe(self):
1161
        """Return a recipe that can be used to replay this search.
1162
        
1163
        The recipe allows reconstruction of the same results at a later date
1164
        without knowing all the found keys. The essential elements are a list
1165
        of keys to start and and to stop at. In order to give reproducible
1166
        results when ghosts are encountered by a search they are automatically
1167
        added to the exclude list (or else ghost filling may alter the
1168
        results).
1169
1170
        :return: A tuple (start_keys_set, exclude_keys_set, revision_count). To
1171
            recreate the results of this search, create a breadth first
1172
            searcher on the same graph starting at start_keys. Then call next()
1173
            (or next_with_ghosts()) repeatedly, and on every result, call
1174
            stop_searching_any on any keys from the exclude_keys set. The
1175
            revision_count value acts as a trivial cross-check - the found
1176
            revisions of the new search should have as many elements as
1177
            revision_count. If it does not, then additional revisions have been
1178
            ghosted since the search was executed the first time and the second
1179
            time.
1180
        """
1181
        return self._recipe
1182
1183
    def get_keys(self):
1184
        """Return the keys found in this search.
1185
1186
        :return: A set of keys.
1187
        """
1188
        return self._keys
1189