/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
3350.6.12 by Martin Pool
merge trunk; remove RemoteToOtherFetcher
1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
1185.65.10 by Robert Collins
Rename Controlfiles to LockableFiles.
16
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
17
from bzrlib.lazy_import import lazy_import
18
lazy_import(globals(), """
3224.5.25 by Andrew Bennetts
More minor import tidying suggested by pyflakes.
19
import cStringIO
1740.3.7 by Jelmer Vernooij
Move committer, log, revprops, timestamp and timezone to CommitBuilder.
20
import re
21
import time
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
22
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
23
from bzrlib import (
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
24
    bzrdir,
25
    check,
2745.1.1 by Robert Collins
Add a number of -Devil checkpoints.
26
    debug,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
27
    errors,
2116.4.1 by John Arbash Meinel
Update file and revision id generators.
28
    generate_ids,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
29
    gpg,
30
    graph,
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
31
    lazy_regex,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
32
    lockable_files,
33
    lockdir,
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
34
    lru_cache,
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
35
    osutils,
2018.5.163 by Andrew Bennetts
Deal with various review comments from Robert.
36
    remote,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
37
    revision as _mod_revision,
38
    symbol_versioning,
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
39
    tsort,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
40
    ui,
3831.2.1 by Andrew Bennetts
Quick hack to do batching in InterDifferingSerializer. Almost halves the HPSS round-trips fetching pack-0.92-subtree to 1.9-rich-root.
41
    versionedfile,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
42
    )
2520.4.54 by Aaron Bentley
Hang a create_bundle method off repository
43
from bzrlib.bundle import serializer
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
44
from bzrlib.revisiontree import RevisionTree
45
from bzrlib.store.versioned import VersionedFileStore
46
from bzrlib.testament import Testament
47
""")
48
3224.5.25 by Andrew Bennetts
More minor import tidying suggested by pyflakes.
49
from bzrlib import registry
1534.4.28 by Robert Collins
first cut at merge from integration.
50
from bzrlib.decorators import needs_read_lock, needs_write_lock
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
51
from bzrlib.inter import InterObject
1910.2.3 by Aaron Bentley
All tests pass
52
from bzrlib.inventory import Inventory, InventoryDirectory, ROOT_ID
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
53
from bzrlib.symbol_versioning import (
54
        deprecated_method,
3341.2.2 by Alexander Belchenko
Tree.print_file and Repository.print_file are deprecated.
55
        one_one,
56
        one_two,
3511.1.2 by Aaron Bentley
Update text and deprecation symbols.
57
        one_six,
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
58
        )
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
59
from bzrlib.trace import (
60
    log_exception_quietly, note, mutter, mutter_callsite, warning)
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
61
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
62
1904.2.5 by Martin Pool
Fix format warning inside test suite and add test
63
# Old formats display a warning, but only once
64
_deprecation_warning_done = False
65
66
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
67
class CommitBuilder(object):
68
    """Provides an interface to build up a commit.
69
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
70
    This allows describing a tree to be committed without needing to
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
71
    know the internals of the format of the repository.
72
    """
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
73
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
74
    # all clients should supply tree roots.
75
    record_root_entry = True
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
76
    # the default CommitBuilder does not manage trees whose root is versioned.
77
    _versioned_root = False
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
78
2979.2.2 by Robert Collins
Per-file graph heads detection during commit for pack repositories.
79
    def __init__(self, repository, parents, config, timestamp=None,
80
                 timezone=None, committer=None, revprops=None,
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
81
                 revision_id=None):
82
        """Initiate a CommitBuilder.
83
84
        :param repository: Repository to commit to.
85
        :param parents: Revision ids of the parents of the new revision.
86
        :param config: Configuration to use.
87
        :param timestamp: Optional timestamp recorded for commit.
88
        :param timezone: Optional timezone for timestamp.
89
        :param committer: Optional committer to set for commit.
90
        :param revprops: Optional dictionary of revision properties.
91
        :param revision_id: Optional revision id.
92
        """
93
        self._config = config
94
95
        if committer is None:
96
            self._committer = self._config.username()
97
        else:
98
            self._committer = committer
99
100
        self.new_inventory = Inventory(None)
2858.2.1 by Martin Pool
Remove most calls to safe_file_id and safe_revision_id.
101
        self._new_revision_id = revision_id
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
102
        self.parents = parents
103
        self.repository = repository
104
105
        self._revprops = {}
106
        if revprops is not None:
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
107
            self._validate_revprops(revprops)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
108
            self._revprops.update(revprops)
109
110
        if timestamp is None:
111
            timestamp = time.time()
112
        # Restrict resolution to 1ms
113
        self._timestamp = round(timestamp, 3)
114
115
        if timezone is None:
116
            self._timezone = osutils.local_time_offset()
117
        else:
118
            self._timezone = int(timezone)
119
120
        self._generate_revision_if_needed()
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
121
        self.__heads = graph.HeadsCache(repository.get_graph()).heads
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
122
        self._basis_delta = []
123
        # API compatibility, older code that used CommitBuilder did not call
124
        # .record_delete(), which means the delta that is computed would not be
125
        # valid. Callers that will call record_delete() should call
126
        # .will_record_deletes() to indicate that.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
127
        self._recording_deletes = False
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
128
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
129
    def _validate_unicode_text(self, text, context):
130
        """Verify things like commit messages don't have bogus characters."""
131
        if '\r' in text:
132
            raise ValueError('Invalid value for %s: %r' % (context, text))
133
134
    def _validate_revprops(self, revprops):
135
        for key, value in revprops.iteritems():
136
            # We know that the XML serializers do not round trip '\r'
137
            # correctly, so refuse to accept them
3831.1.5 by John Arbash Meinel
It seems we have some direct tests that don't use strings and expect a value error as well.
138
            if not isinstance(value, basestring):
139
                raise ValueError('revision property (%s) is not a valid'
140
                                 ' (unicode) string: %r' % (key, value))
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
141
            self._validate_unicode_text(value,
142
                                        'revision property (%s)' % (key,))
143
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
144
    def commit(self, message):
145
        """Make the actual commit.
146
147
        :return: The revision id of the recorded revision.
148
        """
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
149
        self._validate_unicode_text(message, 'commit message')
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
150
        rev = _mod_revision.Revision(
151
                       timestamp=self._timestamp,
152
                       timezone=self._timezone,
153
                       committer=self._committer,
154
                       message=message,
155
                       inventory_sha1=self.inv_sha1,
156
                       revision_id=self._new_revision_id,
157
                       properties=self._revprops)
158
        rev.parent_ids = self.parents
159
        self.repository.add_revision(self._new_revision_id, rev,
160
            self.new_inventory, self._config)
161
        self.repository.commit_write_group()
162
        return self._new_revision_id
163
164
    def abort(self):
165
        """Abort the commit that is being built.
166
        """
167
        self.repository.abort_write_group()
168
169
    def revision_tree(self):
170
        """Return the tree that was just committed.
171
172
        After calling commit() this can be called to get a RevisionTree
173
        representing the newly committed tree. This is preferred to
174
        calling Repository.revision_tree() because that may require
175
        deserializing the inventory, while we already have a copy in
176
        memory.
177
        """
178
        return RevisionTree(self.repository, self.new_inventory,
179
                            self._new_revision_id)
180
181
    def finish_inventory(self):
182
        """Tell the builder that the inventory is finished."""
183
        if self.new_inventory.root is None:
2903.2.9 by Martin Pool
Review cleanups, mostly documentation
184
            raise AssertionError('Root entry should be supplied to'
3535.5.2 by John Arbash Meinel
AssertionError doesn't take extra arguments.
185
                ' record_entry_contents, as of bzr 0.10.')
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
186
            self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
187
        self.new_inventory.revision_id = self._new_revision_id
188
        self.inv_sha1 = self.repository.add_inventory(
189
            self._new_revision_id,
190
            self.new_inventory,
191
            self.parents
192
            )
193
194
    def _gen_revision_id(self):
195
        """Return new revision-id."""
196
        return generate_ids.gen_revision_id(self._config.username(),
197
                                            self._timestamp)
198
199
    def _generate_revision_if_needed(self):
200
        """Create a revision id if None was supplied.
201
        
202
        If the repository can not support user-specified revision ids
203
        they should override this function and raise CannotSetRevisionId
204
        if _new_revision_id is not None.
205
206
        :raises: CannotSetRevisionId
207
        """
208
        if self._new_revision_id is None:
209
            self._new_revision_id = self._gen_revision_id()
210
            self.random_revid = True
211
        else:
212
            self.random_revid = False
213
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
214
    def _heads(self, file_id, revision_ids):
2979.2.1 by Robert Collins
Make it possible for different commit builders to override heads().
215
        """Calculate the graph heads for revision_ids in the graph of file_id.
216
217
        This can use either a per-file graph or a global revision graph as we
218
        have an identity relationship between the two graphs.
219
        """
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
220
        return self.__heads(revision_ids)
2979.2.1 by Robert Collins
Make it possible for different commit builders to override heads().
221
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
222
    def _check_root(self, ie, parent_invs, tree):
223
        """Helper for record_entry_contents.
224
225
        :param ie: An entry being added.
226
        :param parent_invs: The inventories of the parent revisions of the
227
            commit.
228
        :param tree: The tree that is being committed.
229
        """
2871.1.2 by Robert Collins
* ``CommitBuilder.record_entry_contents`` now requires the root entry of a
230
        # In this revision format, root entries have no knit or weave When
231
        # serializing out to disk and back in root.revision is always
232
        # _new_revision_id
233
        ie.revision = self._new_revision_id
2818.3.1 by Robert Collins
Change CommitBuilder factory delegation to allow simple declaration.
234
2871.1.4 by Robert Collins
Merge bzr.dev.
235
    def _get_delta(self, ie, basis_inv, path):
236
        """Get a delta against the basis inventory for ie."""
237
        if ie.file_id not in basis_inv:
238
            # add
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
239
            result = (None, path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
240
            self._basis_delta.append(result)
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
241
            return result
2871.1.4 by Robert Collins
Merge bzr.dev.
242
        elif ie != basis_inv[ie.file_id]:
243
            # common but altered
244
            # TODO: avoid tis id2path call.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
245
            result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
246
            self._basis_delta.append(result)
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
247
            return result
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
248
        else:
2871.1.4 by Robert Collins
Merge bzr.dev.
249
            # common, unaltered
250
            return None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
251
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
252
    def get_basis_delta(self):
253
        """Return the complete inventory delta versus the basis inventory.
254
255
        This has been built up with the calls to record_delete and
256
        record_entry_contents. The client must have already called
257
        will_record_deletes() to indicate that they will be generating a
258
        complete delta.
259
260
        :return: An inventory delta, suitable for use with apply_delta, or
261
            Repository.add_inventory_by_delta, etc.
262
        """
263
        if not self._recording_deletes:
264
            raise AssertionError("recording deletes not activated.")
265
        return self._basis_delta
266
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
267
    def record_delete(self, path, file_id):
268
        """Record that a delete occured against a basis tree.
269
270
        This is an optional API - when used it adds items to the basis_delta
271
        being accumulated by the commit builder. It cannot be called unless the
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
272
        method will_record_deletes() has been called to inform the builder that
273
        a delta is being supplied.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
274
275
        :param path: The path of the thing deleted.
276
        :param file_id: The file id that was deleted.
277
        """
278
        if not self._recording_deletes:
279
            raise AssertionError("recording deletes not activated.")
3879.2.5 by John Arbash Meinel
Change record_delete() to return the delta.
280
        delta = (path, None, file_id, None)
281
        self._basis_delta.append(delta)
282
        return delta
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
283
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
284
    def will_record_deletes(self):
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
285
        """Tell the commit builder that deletes are being notified.
286
287
        This enables the accumulation of an inventory delta; for the resulting
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
288
        commit to be valid, deletes against the basis MUST be recorded via
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
289
        builder.record_delete().
290
        """
291
        self._recording_deletes = True
292
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
293
    def record_entry_contents(self, ie, parent_invs, path, tree,
294
        content_summary):
295
        """Record the content of ie from tree into the commit if needed.
296
297
        Side effect: sets ie.revision when unchanged
298
299
        :param ie: An inventory entry present in the commit.
300
        :param parent_invs: The inventories of the parent revisions of the
301
            commit.
302
        :param path: The path the entry is at in the tree.
303
        :param tree: The tree which contains this entry and should be used to 
304
            obtain content.
305
        :param content_summary: Summary data from the tree about the paths
306
            content - stat, length, exec, sha/link target. This is only
307
            accessed when the entry has a revision of None - that is when it is
308
            a candidate to commit.
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
309
        :return: A tuple (change_delta, version_recorded, fs_hash).
310
            change_delta is an inventory_delta change for this entry against
311
            the basis tree of the commit, or None if no change occured against
312
            the basis tree.
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
313
            version_recorded is True if a new version of the entry has been
314
            recorded. For instance, committing a merge where a file was only
315
            changed on the other side will return (delta, False).
3709.3.3 by Robert Collins
NEWS for the record_entry_contents change.
316
            fs_hash is either None, or the hash details for the path (currently
317
            a tuple of the contents sha1 and the statvalue returned by
318
            tree.get_file_with_stat()).
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
319
        """
320
        if self.new_inventory.root is None:
2871.1.2 by Robert Collins
* ``CommitBuilder.record_entry_contents`` now requires the root entry of a
321
            if ie.parent_id is not None:
322
                raise errors.RootMissing()
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
323
            self._check_root(ie, parent_invs, tree)
324
        if ie.revision is None:
325
            kind = content_summary[0]
326
        else:
327
            # ie is carried over from a prior commit
328
            kind = ie.kind
329
        # XXX: repository specific check for nested tree support goes here - if
330
        # the repo doesn't want nested trees we skip it ?
331
        if (kind == 'tree-reference' and
332
            not self.repository._format.supports_tree_reference):
333
            # mismatch between commit builder logic and repository:
334
            # this needs the entry creation pushed down into the builder.
2776.4.18 by Robert Collins
Review feedback.
335
            raise NotImplementedError('Missing repository subtree support.')
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
336
        self.new_inventory.add(ie)
337
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
338
        # TODO: slow, take it out of the inner loop.
339
        try:
340
            basis_inv = parent_invs[0]
341
        except IndexError:
342
            basis_inv = Inventory(root_id=None)
343
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
344
        # ie.revision is always None if the InventoryEntry is considered
2776.4.13 by Robert Collins
Merge bzr.dev.
345
        # for committing. We may record the previous parents revision if the
346
        # content is actually unchanged against a sole head.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
347
        if ie.revision is not None:
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
348
            if not self._versioned_root and path == '':
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
349
                # repositories that do not version the root set the root's
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
350
                # revision to the new commit even when no change occurs (more
351
                # specifically, they do not record a revision on the root; and
352
                # the rev id is assigned to the root during deserialisation -
353
                # this masks when a change may have occurred against the basis.
354
                # To match this we always issue a delta, because the revision
355
                # of the root will always be changing.
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
356
                if ie.file_id in basis_inv:
357
                    delta = (basis_inv.id2path(ie.file_id), path,
358
                        ie.file_id, ie)
359
                else:
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
360
                    # add
361
                    delta = (None, path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
362
                self._basis_delta.append(delta)
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
363
                return delta, False, None
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
364
            else:
365
                # we don't need to commit this, because the caller already
366
                # determined that an existing revision of this file is
3619.1.1 by Robert Collins
Tighten up the handling of carried-over inventory entries.
367
                # appropriate. If its not being considered for committing then
368
                # it and all its parents to the root must be unaltered so
369
                # no-change against the basis.
370
                if ie.revision == self._new_revision_id:
371
                    raise AssertionError("Impossible situation, a skipped "
3619.1.2 by Robert Collins
Review feedback.
372
                        "inventory entry (%r) claims to be modified in this "
373
                        "commit (%r).", (ie, self._new_revision_id))
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
374
                return None, False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
375
        # XXX: Friction: parent_candidates should return a list not a dict
376
        #      so that we don't have to walk the inventories again.
377
        parent_candiate_entries = ie.parent_candidates(parent_invs)
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
378
        head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
379
        heads = []
380
        for inv in parent_invs:
381
            if ie.file_id in inv:
382
                old_rev = inv[ie.file_id].revision
383
                if old_rev in head_set:
384
                    heads.append(inv[ie.file_id].revision)
385
                    head_set.remove(inv[ie.file_id].revision)
386
387
        store = False
388
        # now we check to see if we need to write a new record to the
389
        # file-graph.
390
        # We write a new entry unless there is one head to the ancestors, and
391
        # the kind-derived content is unchanged.
392
393
        # Cheapest check first: no ancestors, or more the one head in the
394
        # ancestors, we write a new node.
395
        if len(heads) != 1:
396
            store = True
397
        if not store:
398
            # There is a single head, look it up for comparison
399
            parent_entry = parent_candiate_entries[heads[0]]
400
            # if the non-content specific data has changed, we'll be writing a
401
            # node:
402
            if (parent_entry.parent_id != ie.parent_id or
403
                parent_entry.name != ie.name):
404
                store = True
405
        # now we need to do content specific checks:
406
        if not store:
407
            # if the kind changed the content obviously has
408
            if kind != parent_entry.kind:
409
                store = True
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
410
        # Stat cache fingerprint feedback for the caller - None as we usually
411
        # don't generate one.
412
        fingerprint = None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
413
        if kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
414
            if content_summary[2] is None:
415
                raise ValueError("Files must not have executable = None")
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
416
            if not store:
417
                if (# if the file length changed we have to store:
418
                    parent_entry.text_size != content_summary[1] or
419
                    # if the exec bit has changed we have to store:
420
                    parent_entry.executable != content_summary[2]):
421
                    store = True
422
                elif parent_entry.text_sha1 == content_summary[3]:
423
                    # all meta and content is unchanged (using a hash cache
424
                    # hit to check the sha)
425
                    ie.revision = parent_entry.revision
426
                    ie.text_size = parent_entry.text_size
427
                    ie.text_sha1 = parent_entry.text_sha1
428
                    ie.executable = parent_entry.executable
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
429
                    return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
430
                else:
431
                    # Either there is only a hash change(no hash cache entry,
432
                    # or same size content change), or there is no change on
433
                    # this file at all.
2776.4.19 by Robert Collins
Final review tweaks.
434
                    # Provide the parent's hash to the store layer, so that the
435
                    # content is unchanged we will not store a new node.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
436
                    nostore_sha = parent_entry.text_sha1
437
            if store:
2776.4.18 by Robert Collins
Review feedback.
438
                # We want to record a new node regardless of the presence or
439
                # absence of a content change in the file.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
440
                nostore_sha = None
2776.4.18 by Robert Collins
Review feedback.
441
            ie.executable = content_summary[2]
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
442
            file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
443
            try:
444
                lines = file_obj.readlines()
445
            finally:
446
                file_obj.close()
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
447
            try:
448
                ie.text_sha1, ie.text_size = self._add_text_to_weave(
449
                    ie.file_id, lines, heads, nostore_sha)
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
450
                # Let the caller know we generated a stat fingerprint.
451
                fingerprint = (ie.text_sha1, stat_value)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
452
            except errors.ExistingContent:
2776.4.18 by Robert Collins
Review feedback.
453
                # Turns out that the file content was unchanged, and we were
454
                # only going to store a new node if it was changed. Carry over
455
                # the entry.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
456
                ie.revision = parent_entry.revision
457
                ie.text_size = parent_entry.text_size
458
                ie.text_sha1 = parent_entry.text_sha1
459
                ie.executable = parent_entry.executable
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
460
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
461
        elif kind == 'directory':
462
            if not store:
463
                # all data is meta here, nothing specific to directory, so
464
                # carry over:
465
                ie.revision = parent_entry.revision
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
466
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
467
            lines = []
468
            self._add_text_to_weave(ie.file_id, lines, heads, None)
469
        elif kind == 'symlink':
470
            current_link_target = content_summary[3]
471
            if not store:
2776.4.18 by Robert Collins
Review feedback.
472
                # symlink target is not generic metadata, check if it has
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
473
                # changed.
474
                if current_link_target != parent_entry.symlink_target:
475
                    store = True
476
            if not store:
477
                # unchanged, carry over.
478
                ie.revision = parent_entry.revision
479
                ie.symlink_target = parent_entry.symlink_target
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
480
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
481
            ie.symlink_target = current_link_target
482
            lines = []
483
            self._add_text_to_weave(ie.file_id, lines, heads, None)
484
        elif kind == 'tree-reference':
485
            if not store:
486
                if content_summary[3] != parent_entry.reference_revision:
487
                    store = True
488
            if not store:
489
                # unchanged, carry over.
490
                ie.reference_revision = parent_entry.reference_revision
491
                ie.revision = parent_entry.revision
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
492
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
493
            ie.reference_revision = content_summary[3]
494
            lines = []
495
            self._add_text_to_weave(ie.file_id, lines, heads, None)
496
        else:
497
            raise NotImplementedError('unknown kind')
498
        ie.revision = self._new_revision_id
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
499
        return self._get_delta(ie, basis_inv, path), True, fingerprint
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
500
501
    def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
2592.3.136 by Robert Collins
Merge bzr.dev.
502
        # Note: as we read the content directly from the tree, we know its not
503
        # been turned into unicode or badly split - but a broken tree
504
        # implementation could give us bad output from readlines() so this is
505
        # not a guarantee of safety. What would be better is always checking
506
        # the content during test suite execution. RBC 20070912
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
507
        parent_keys = tuple((file_id, parent) for parent in parents)
508
        return self.repository.texts.add_lines(
509
            (file_id, self._new_revision_id), parent_keys, new_lines,
3316.2.12 by Robert Collins
Catch some extra deprecated calls.
510
            nostore_sha=nostore_sha, random_id=self.random_revid,
511
            check_content=False)[0:2]
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
512
513
514
class RootCommitBuilder(CommitBuilder):
515
    """This commitbuilder actually records the root id"""
516
    
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
517
    # the root entry gets versioned properly by this builder.
2840.1.1 by Ian Clatworthy
faster pointless commit detection (Robert Collins)
518
    _versioned_root = True
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
519
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
520
    def _check_root(self, ie, parent_invs, tree):
521
        """Helper for record_entry_contents.
522
523
        :param ie: An entry being added.
524
        :param parent_invs: The inventories of the parent revisions of the
525
            commit.
526
        :param tree: The tree that is being committed.
527
        """
528
529
2220.2.3 by Martin Pool
Add tag: revision namespace.
530
######################################################################
531
# Repositories
532
1185.66.5 by Aaron Bentley
Renamed RevisionStorage to Repository
533
class Repository(object):
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
534
    """Repository holding history for one or more branches.
535
536
    The repository holds and retrieves historical information including
537
    revisions and file history.  It's normally accessed only by the Branch,
538
    which views a particular line of development through that history.
539
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
540
    The Repository builds on top of some byte storage facilies (the revisions,
541
    signatures, inventories and texts attributes) and a Transport, which
542
    respectively provide byte storage and a means to access the (possibly
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
543
    remote) disk.
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
544
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
545
    The byte storage facilities are addressed via tuples, which we refer to
546
    as 'keys' throughout the code base. Revision_keys, inventory_keys and
547
    signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
548
    (file_id, revision_id). We use this interface because it allows low
549
    friction with the underlying code that implements disk indices, network
550
    encoding and other parts of bzrlib.
551
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
552
    :ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
553
        the serialised revisions for the repository. This can be used to obtain
554
        revision graph information or to access raw serialised revisions.
555
        The result of trying to insert data into the repository via this store
556
        is undefined: it should be considered read-only except for implementors
557
        of repositories.
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
558
    :ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
559
        the serialised signatures for the repository. This can be used to
560
        obtain access to raw serialised signatures.  The result of trying to
561
        insert data into the repository via this store is undefined: it should
562
        be considered read-only except for implementors of repositories.
563
    :ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
564
        the serialised inventories for the repository. This can be used to
565
        obtain unserialised inventories.  The result of trying to insert data
566
        into the repository via this store is undefined: it should be
567
        considered read-only except for implementors of repositories.
568
    :ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
569
        texts of files and directories for the repository. This can be used to
570
        obtain file texts or file graphs. Note that Repository.iter_file_bytes
571
        is usually a better interface for accessing file texts.
572
        The result of trying to insert data into the repository via this store
573
        is undefined: it should be considered read-only except for implementors
574
        of repositories.
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
575
    :ivar _transport: Transport for file access to repository, typically
576
        pointing to .bzr/repository.
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
577
    """
1185.65.17 by Robert Collins
Merge from integration, mode-changes are broken.
578
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
579
    # What class to use for a CommitBuilder. Often its simpler to change this
580
    # in a Repository class subclass rather than to override
581
    # get_commit_builder.
582
    _commit_builder_class = CommitBuilder
583
    # The search regex used by xml based repositories to determine what things
584
    # where changed in a single commit.
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
585
    _file_ids_altered_regex = lazy_regex.lazy_compile(
586
        r'file_id="(?P<file_id>[^"]+)"'
2776.4.6 by Robert Collins
Fixup various commit test failures falling out from the other commit changes.
587
        r'.* revision="(?P<revision_id>[^"]+)"'
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
588
        )
589
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
590
    def abort_write_group(self, suppress_errors=False):
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
591
        """Commit the contents accrued within the current write group.
592
3825.4.6 by Andrew Bennetts
Document the suppress_errors flag in the docstring.
593
        :param suppress_errors: if true, abort_write_group will catch and log
594
            unexpected errors that happen during the abort, rather than
595
            allowing them to propagate.  Defaults to False.
596
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
597
        :seealso: start_write_group.
598
        """
599
        if self._write_group is not self.get_transaction():
600
            # has an unlock or relock occured ?
601
            raise errors.BzrError('mismatched lock context and write group.')
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
602
        try:
603
            self._abort_write_group()
604
        except Exception, exc:
605
            self._write_group = None
606
            if not suppress_errors:
607
                raise
608
            mutter('abort_write_group failed')
609
            log_exception_quietly()
610
            note('bzr: ERROR (ignored): %s', exc)
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
611
        self._write_group = None
612
613
    def _abort_write_group(self):
614
        """Template method for per-repository write group cleanup.
615
        
616
        This is called during abort before the write group is considered to be 
617
        finished and should cleanup any internal state accrued during the write
618
        group. There is no requirement that data handed to the repository be
619
        *not* made available - this is not a rollback - but neither should any
620
        attempt be made to ensure that data added is fully commited. Abort is
621
        invoked when an error has occured so futher disk or network operations
622
        may not be possible or may error and if possible should not be
623
        attempted.
624
        """
625
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
626
    def add_fallback_repository(self, repository):
627
        """Add a repository to use for looking up data not held locally.
628
        
629
        :param repository: A repository.
630
        """
631
        if not self._format.supports_external_lookups:
632
            raise errors.UnstackableRepositoryFormat(self._format, self.base)
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
633
        self._check_fallback_repository(repository)
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
634
        self._fallback_repositories.append(repository)
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
635
        self.texts.add_fallback_versioned_files(repository.texts)
636
        self.inventories.add_fallback_versioned_files(repository.inventories)
637
        self.revisions.add_fallback_versioned_files(repository.revisions)
638
        self.signatures.add_fallback_versioned_files(repository.signatures)
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
639
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
640
    def _check_fallback_repository(self, repository):
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
641
        """Check that this repository can fallback to repository safely.
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
642
643
        Raise an error if not.
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
644
        
645
        :param repository: A repository to fallback to.
646
        """
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
647
        return InterRepository._assert_same_model(self, repository)
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
648
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
649
    def add_inventory(self, revision_id, inv, parents):
650
        """Add the inventory inv to the repository as revision_id.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
651
        
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
652
        :param parents: The revision ids of the parents that revision_id
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
653
                        is known to have and are in the repository already.
654
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
655
        :returns: The validator(which is a sha1 digest, though what is sha'd is
656
            repository format specific) of the serialized inventory.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
657
        """
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
658
        if not self.is_in_write_group():
659
            raise AssertionError("%r not in write group" % (self,))
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
660
        _mod_revision.check_not_reserved_id(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
661
        if not (inv.revision_id is None or inv.revision_id == revision_id):
662
            raise AssertionError(
663
                "Mismatch between inventory revision"
664
                " id and insertion revid (%r, %r)"
665
                % (inv.revision_id, revision_id))
666
        if inv.root is None:
667
            raise AssertionError()
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
668
        inv_lines = self._serialise_inventory_to_lines(inv)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
669
        return self._inventory_add_lines(revision_id, parents,
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
670
            inv_lines, check_content=False)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
671
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
672
    def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
673
                               parents):
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
674
        """Add a new inventory expressed as a delta against another revision.
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
675
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
676
        :param basis_revision_id: The inventory id the delta was created
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
677
            against. (This does not have to be a direct parent.)
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
678
        :param delta: The inventory delta (see Inventory.apply_delta for
679
            details).
680
        :param new_revision_id: The revision id that the inventory is being
681
            added for.
682
        :param parents: The revision ids of the parents that revision_id is
683
            known to have and are in the repository already. These are supplied
684
            for repositories that depend on the inventory graph for revision
685
            graph access, as well as for those that pun ancestry with delta
686
            compression.
687
688
        :returns: The validator(which is a sha1 digest, though what is sha'd is
689
            repository format specific) of the serialized inventory.
690
        """
691
        if not self.is_in_write_group():
692
            raise AssertionError("%r not in write group" % (self,))
693
        _mod_revision.check_not_reserved_id(new_revision_id)
694
        basis_tree = self.revision_tree(basis_revision_id)
695
        basis_tree.lock_read()
696
        try:
697
            # Note that this mutates the inventory of basis_tree, which not all
698
            # inventory implementations may support: A better idiom would be to
699
            # return a new inventory, but as there is no revision tree cache in
700
            # repository this is safe for now - RBC 20081013
701
            basis_inv = basis_tree.inventory
702
            basis_inv.apply_delta(delta)
703
            basis_inv.revision_id = new_revision_id
704
            return self.add_inventory(new_revision_id, basis_inv, parents)
705
        finally:
706
            basis_tree.unlock()
707
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
708
    def _inventory_add_lines(self, revision_id, parents, lines,
2805.6.7 by Robert Collins
Review feedback.
709
        check_content=True):
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
710
        """Store lines in inv_vf and return the sha1 of the inventory."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
711
        parents = [(parent,) for parent in parents]
712
        return self.inventories.add_lines((revision_id,), parents, lines,
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
713
            check_content=check_content)[0]
1740.3.6 by Jelmer Vernooij
Move inventory writing to the commit builder.
714
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
715
    def add_revision(self, revision_id, rev, inv=None, config=None):
716
        """Add rev to the revision store as revision_id.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
717
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
718
        :param revision_id: the revision id to use.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
719
        :param rev: The revision object.
720
        :param inv: The inventory for the revision. if None, it will be looked
721
                    up in the inventory storer
722
        :param config: If None no digital signature will be created.
723
                       If supplied its signature_needed method will be used
724
                       to determine if a signature should be made.
725
        """
2249.5.13 by John Arbash Meinel
Finish auditing Repository, and fix generate_ids to always generate utf8 ids.
726
        # TODO: jam 20070210 Shouldn't we check rev.revision_id and
727
        #       rev.parent_ids?
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
728
        _mod_revision.check_not_reserved_id(revision_id)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
729
        if config is not None and config.signature_needed():
730
            if inv is None:
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
731
                inv = self.get_inventory(revision_id)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
732
            plaintext = Testament(rev, inv).as_short_text()
733
            self.store_revision_signature(
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
734
                gpg.GPGStrategy(config), plaintext, revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
735
        # check inventory present
736
        if not self.inventories.get_parent_map([(revision_id,)]):
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
737
            if inv is None:
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
738
                raise errors.WeaveRevisionNotPresent(revision_id,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
739
                                                     self.inventories)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
740
            else:
741
                # yes, this is not suitable for adding with ghosts.
3380.1.6 by Aaron Bentley
Ensure fetching munges sha1s
742
                rev.inventory_sha1 = self.add_inventory(revision_id, inv,
3305.1.1 by Jelmer Vernooij
Make sure that specifying the inv= argument to add_revision() sets the
743
                                                        rev.parent_ids)
3380.1.6 by Aaron Bentley
Ensure fetching munges sha1s
744
        else:
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
745
            key = (revision_id,)
746
            rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
747
        self._add_revision(rev)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
748
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
749
    def _add_revision(self, revision):
750
        text = self._serializer.write_revision_to_string(revision)
751
        key = (revision.revision_id,)
752
        parents = tuple((parent,) for parent in revision.parent_ids)
753
        self.revisions.add_lines(key, parents, osutils.split_lines(text))
2520.4.10 by Aaron Bentley
Enable installation of revisions
754
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
755
    def all_revision_ids(self):
756
        """Returns a list of all the revision ids in the repository. 
757
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
758
        This is conceptually deprecated because code should generally work on
759
        the graph reachable from a particular revision, and ignore any other
760
        revisions that might be present.  There is no direct replacement
761
        method.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
762
        """
2592.3.114 by Robert Collins
More evil mutterings.
763
        if 'evil' in debug.debug_flags:
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
764
            mutter_callsite(2, "all_revision_ids is linear with history.")
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
765
        return self._all_revision_ids()
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
766
767
    def _all_revision_ids(self):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
768
        """Returns a list of all the revision ids in the repository. 
769
770
        These are in as much topological order as the underlying store can 
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
771
        present.
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
772
        """
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
773
        raise NotImplementedError(self._all_revision_ids)
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
774
1687.1.7 by Robert Collins
Teach Repository about break_lock.
775
    def break_lock(self):
776
        """Break a lock if one is present from another instance.
777
778
        Uses the ui factory to ask for confirmation if the lock may be from
779
        an active process.
780
        """
781
        self.control_files.break_lock()
782
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
783
    @needs_read_lock
784
    def _eliminate_revisions_not_present(self, revision_ids):
785
        """Check every revision id in revision_ids to see if we have it.
786
787
        Returns a set of the present revisions.
788
        """
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
789
        result = []
3369.2.1 by John Arbash Meinel
Knit => knit fetching also has some very bad 'for x in revision_ids: has_revision_id()' calls
790
        graph = self.get_graph()
791
        parent_map = graph.get_parent_map(revision_ids)
792
        # The old API returned a list, should this actually be a set?
793
        return parent_map.keys()
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
794
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
795
    @staticmethod
796
    def create(a_bzrdir):
797
        """Construct the current default format repository in a_bzrdir."""
798
        return RepositoryFormat.get_default_format().initialize(a_bzrdir)
799
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
800
    def __init__(self, _format, a_bzrdir, control_files):
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
801
        """instantiate a Repository.
802
803
        :param _format: The format of the repository on disk.
804
        :param a_bzrdir: The BzrDir of the repository.
805
806
        In the future we will have a single api for all stores for
807
        getting file texts, inventories and revisions, then
808
        this construct will accept instances of those things.
809
        """
1608.2.1 by Martin Pool
[merge] Storage filename escaping
810
        super(Repository, self).__init__()
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
811
        self._format = _format
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
812
        # the following are part of the public API for Repository:
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
813
        self.bzrdir = a_bzrdir
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
814
        self.control_files = control_files
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
815
        self._transport = control_files._transport
3407.2.14 by Martin Pool
Remove more cases of getting transport via control_files
816
        self.base = self._transport.base
2671.4.2 by Robert Collins
Review feedback.
817
        # for tests
818
        self._reconcile_does_inventory_gc = True
2745.6.16 by Aaron Bentley
Update from review
819
        self._reconcile_fixes_text_parents = False
2951.1.3 by Robert Collins
Partial support for native reconcile with packs.
820
        self._reconcile_backsup_inventory = True
1563.2.23 by Robert Collins
Add add_revision and get_revision methods to RevisionStore
821
        # not right yet - should be more semantically clear ? 
822
        # 
1608.2.1 by Martin Pool
[merge] Storage filename escaping
823
        # TODO: make sure to construct the right store classes, etc, depending
824
        # on whether escaping is required.
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
825
        self._warn_if_deprecated()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
826
        self._write_group = None
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
827
        # Additional places to query for data.
828
        self._fallback_repositories = []
3565.3.1 by Robert Collins
* The generic fetch code now uses two attributes on Repository objects
829
        # What order should fetch operations request streams in?
3606.7.7 by John Arbash Meinel
Add tests for the fetching behavior.
830
        # The default is unordered as that is the cheapest for an origin to
3606.7.4 by John Arbash Meinel
We don't have to change the default order either.
831
        # provide.
3606.7.7 by John Arbash Meinel
Add tests for the fetching behavior.
832
        self._fetch_order = 'unordered'
3830.3.14 by Martin Pool
Return to setting _fetch_uses_deltas from initializers
833
        # Does this repository use deltas that can be fetched as-deltas ?
834
        # (E.g. knits, where the knit deltas can be transplanted intact.
835
        # We default to False, which will ensure that enough data to get
836
        # a full text out of any fetch stream will be grabbed.
837
        self._fetch_uses_deltas = False
3565.3.4 by Robert Collins
Defer decision to reconcile to the repository being fetched into.
838
        # Should fetch trigger a reconcile after the fetch? Only needed for
839
        # some repository formats that can suffer internal inconsistencies.
840
        self._fetch_reconcile = False
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
841
1668.1.3 by Martin Pool
[patch] use the correct transaction when committing snapshot (Malone: #43959)
842
    def __repr__(self):
2592.4.5 by Martin Pool
Add Repository.base on all repositories.
843
        return '%s(%r)' % (self.__class__.__name__,
844
                           self.base)
1668.1.3 by Martin Pool
[patch] use the correct transaction when committing snapshot (Malone: #43959)
845
2671.1.4 by Andrew Bennetts
Rename is_same_repository to has_same_location, thanks Aaron!
846
    def has_same_location(self, other):
2671.1.3 by Andrew Bennetts
Remove Repository.__eq__/__ne__ methods, replace with is_same_repository method.
847
        """Returns a boolean indicating if this repository is at the same
848
        location as another repository.
849
850
        This might return False even when two repository objects are accessing
851
        the same physical repository via different URLs.
852
        """
2592.3.162 by Robert Collins
Remove some arbitrary differences from bzr.dev.
853
        if self.__class__ is not other.__class__:
854
            return False
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
855
        return (self._transport.base == other._transport.base)
2671.1.1 by Andrew Bennetts
Add support for comparing Repositories with == and != operators.
856
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
857
    def is_in_write_group(self):
858
        """Return True if there is an open write group.
859
860
        :seealso: start_write_group.
861
        """
862
        return self._write_group is not None
863
1694.2.6 by Martin Pool
[merge] bzr.dev
864
    def is_locked(self):
865
        return self.control_files.is_locked()
866
2592.3.188 by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency.
867
    def is_write_locked(self):
868
        """Return True if this object is write locked."""
869
        return self.is_locked() and self.control_files._lock_mode == 'w'
870
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
871
    def lock_write(self, token=None):
872
        """Lock this repository for writing.
2617.6.8 by Robert Collins
Review feedback and documentation.
873
874
        This causes caching within the repository obejct to start accumlating
875
        data during reads, and allows a 'write_group' to be obtained. Write
876
        groups must be used for actual data insertion.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
877
        
878
        :param token: if this is already locked, then lock_write will fail
879
            unless the token matches the existing lock.
880
        :returns: a token if this instance supports tokens, otherwise None.
881
        :raises TokenLockingNotSupported: when a token is given but this
882
            instance doesn't support using token locks.
883
        :raises MismatchedToken: if the specified token doesn't match the token
884
            of the existing lock.
2617.6.8 by Robert Collins
Review feedback and documentation.
885
        :seealso: start_write_group.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
886
2018.5.145 by Andrew Bennetts
Add a brief explanation of what tokens are used for to lock_write docstrings.
887
        A token should be passed in if you know that you have locked the object
888
        some other way, and need to synchronise this object's state with that
889
        fact.
890
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
891
        XXX: this docstring is duplicated in many places, e.g. lockable_files.py
892
        """
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
893
        result = self.control_files.lock_write(token=token)
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
894
        for repo in self._fallback_repositories:
895
            # Writes don't affect fallback repos
896
            repo.lock_read()
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
897
        self._refresh_data()
898
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
899
900
    def lock_read(self):
1553.5.55 by Martin Pool
[revert] broken changes
901
        self.control_files.lock_read()
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
902
        for repo in self._fallback_repositories:
903
            repo.lock_read()
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
904
        self._refresh_data()
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
905
1694.2.6 by Martin Pool
[merge] bzr.dev
906
    def get_physical_lock_status(self):
907
        return self.control_files.get_physical_lock_status()
1624.3.36 by Olaf Conradi
Rename is_transport_locked() to get_physical_lock_status() as the
908
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
909
    def leave_lock_in_place(self):
910
        """Tell this repository not to release the physical lock when this
911
        object is unlocked.
2018.5.76 by Andrew Bennetts
Testing that repository.{dont_,}leave_lock_in_place raises NotImplementedError if lock_write returns None.
912
        
913
        If lock_write doesn't return a token, then this method is not supported.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
914
        """
915
        self.control_files.leave_in_place()
916
917
    def dont_leave_lock_in_place(self):
918
        """Tell this repository to release the physical lock when this
919
        object is unlocked, even if it didn't originally acquire it.
2018.5.76 by Andrew Bennetts
Testing that repository.{dont_,}leave_lock_in_place raises NotImplementedError if lock_write returns None.
920
921
        If lock_write doesn't return a token, then this method is not supported.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
922
        """
923
        self.control_files.dont_leave_in_place()
924
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
925
    @needs_read_lock
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
926
    def gather_stats(self, revid=None, committers=None):
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
927
        """Gather statistics from a revision id.
928
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
929
        :param revid: The revision id to gather statistics from, if None, then
930
            no revision specific statistics are gathered.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
931
        :param committers: Optional parameter controlling whether to grab
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
932
            a count of committers from the revision specific statistics.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
933
        :return: A dictionary of statistics. Currently this contains:
934
            committers: The number of committers if requested.
935
            firstrev: A tuple with timestamp, timezone for the penultimate left
936
                most ancestor of revid, if revid is not the NULL_REVISION.
937
            latestrev: A tuple with timestamp, timezone for revid, if revid is
938
                not the NULL_REVISION.
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
939
            revisions: The total revision count in the repository.
940
            size: An estimate disk size of the repository in bytes.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
941
        """
942
        result = {}
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
943
        if revid and committers:
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
944
            result['committers'] = 0
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
945
        if revid and revid != _mod_revision.NULL_REVISION:
946
            if committers:
947
                all_committers = set()
948
            revisions = self.get_ancestry(revid)
949
            # pop the leading None
950
            revisions.pop(0)
951
            first_revision = None
952
            if not committers:
953
                # ignore the revisions in the middle - just grab first and last
954
                revisions = revisions[0], revisions[-1]
955
            for revision in self.get_revisions(revisions):
956
                if not first_revision:
957
                    first_revision = revision
958
                if committers:
959
                    all_committers.add(revision.committer)
960
            last_revision = revision
961
            if committers:
962
                result['committers'] = len(all_committers)
963
            result['firstrev'] = (first_revision.timestamp,
964
                first_revision.timezone)
965
            result['latestrev'] = (last_revision.timestamp,
966
                last_revision.timezone)
967
968
        # now gather global repository information
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
969
        # XXX: This is available for many repos regardless of listability.
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
970
        if self.bzrdir.root_transport.listable():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
971
            # XXX: do we want to __define len__() ?
3350.6.10 by Martin Pool
VersionedFiles review cleanups
972
            # Maybe the versionedfiles object should provide a different
973
            # method to get the number of keys.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
974
            result['revisions'] = len(self.revisions.keys())
975
            # result['size'] = t
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
976
        return result
977
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
978
    def find_branches(self, using=False):
979
        """Find branches underneath this repository.
980
3140.1.7 by Aaron Bentley
Update docs
981
        This will include branches inside other branches.
982
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
983
        :param using: If True, list only branches using this repository.
984
        """
3140.1.9 by Aaron Bentley
Optimize find_branches for standalone repositories
985
        if using and not self.is_shared():
986
            try:
987
                return [self.bzrdir.open_branch()]
988
            except errors.NotBranchError:
989
                return []
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
990
        class Evaluator(object):
991
992
            def __init__(self):
993
                self.first_call = True
994
995
            def __call__(self, bzrdir):
996
                # On the first call, the parameter is always the bzrdir
997
                # containing the current repo.
998
                if not self.first_call:
999
                    try:
1000
                        repository = bzrdir.open_repository()
1001
                    except errors.NoRepositoryPresent:
1002
                        pass
1003
                    else:
1004
                        return False, (None, repository)
1005
                self.first_call = False
1006
                try:
1007
                    value = (bzrdir.open_branch(), None)
1008
                except errors.NotBranchError:
1009
                    value = (None, None)
1010
                return True, value
1011
1012
        branches = []
1013
        for branch, repository in bzrdir.BzrDir.find_bzrdirs(
1014
                self.bzrdir.root_transport, evaluate=Evaluator()):
1015
            if branch is not None:
1016
                branches.append(branch)
1017
            if not using and repository is not None:
1018
                branches.extend(repository.find_branches())
1019
        return branches
1020
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1021
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
1022
    def search_missing_revision_ids(self, other, revision_id=None, find_ghosts=True):
1023
        """Return the revision ids that other has that this does not.
1024
        
1025
        These are returned in topological order.
1026
1027
        revision_id: only return revision ids included by revision_id.
1028
        """
1029
        return InterRepository.get(other, self).search_missing_revision_ids(
1030
            revision_id, find_ghosts)
1031
3341.2.2 by Alexander Belchenko
Tree.print_file and Repository.print_file are deprecated.
1032
    @deprecated_method(one_two)
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
1033
    @needs_read_lock
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
1034
    def missing_revision_ids(self, other, revision_id=None, find_ghosts=True):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1035
        """Return the revision ids that other has that this does not.
1036
        
1037
        These are returned in topological order.
1038
1039
        revision_id: only return revision ids included by revision_id.
1040
        """
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
1041
        keys =  self.search_missing_revision_ids(
1042
            other, revision_id, find_ghosts).get_keys()
1043
        other.lock_read()
1044
        try:
1045
            parents = other.get_graph().get_parent_map(keys)
1046
        finally:
1047
            other.unlock()
1048
        return tsort.topo_sort(parents)
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1049
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1050
    @staticmethod
1051
    def open(base):
1052
        """Open the repository rooted at base.
1053
1054
        For instance, if the repository is at URL/.bzr/repository,
1055
        Repository.open(URL) -> a Repository instance.
1056
        """
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1057
        control = bzrdir.BzrDir.open(base)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1058
        return control.open_repository()
1059
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1060
    def copy_content_into(self, destination, revision_id=None):
1534.6.6 by Robert Collins
Move find_repository to bzrdir, its not quite ideal there but its simpler and until someone chooses to vary the search by branch type its completely sufficient.
1061
        """Make a complete copy of the content in self into destination.
1062
        
1063
        This is a destructive operation! Do not use it on existing 
1064
        repositories.
1065
        """
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1066
        return InterRepository.get(self, destination).copy_content(revision_id)
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1067
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1068
    def commit_write_group(self):
1069
        """Commit the contents accrued within the current write group.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1070
1071
        :seealso: start_write_group.
1072
        """
1073
        if self._write_group is not self.get_transaction():
1074
            # has an unlock or relock occured ?
2592.3.38 by Robert Collins
All experimental format tests passing again.
1075
            raise errors.BzrError('mismatched lock context %r and '
1076
                'write group %r.' %
1077
                (self.get_transaction(), self._write_group))
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1078
        self._commit_write_group()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1079
        self._write_group = None
1080
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1081
    def _commit_write_group(self):
1082
        """Template method for per-repository write group cleanup.
1083
        
1084
        This is called before the write group is considered to be 
1085
        finished and should ensure that all data handed to the repository
1086
        for writing during the write group is safely committed (to the 
1087
        extent possible considering file system caching etc).
1088
        """
1089
2949.1.1 by Robert Collins
Change Repository.fetch to provide a find_ghosts parameter which triggers ghost filling.
1090
    def fetch(self, source, revision_id=None, pb=None, find_ghosts=False):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1091
        """Fetch the content required to construct revision_id from source.
1092
1093
        If revision_id is None all content is copied.
2949.1.1 by Robert Collins
Change Repository.fetch to provide a find_ghosts parameter which triggers ghost filling.
1094
        :param find_ghosts: Find and copy revisions in the source that are
1095
            ghosts in the target (and not reachable directly by walking out to
1096
            the first-present revision in target from revision_id).
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1097
        """
2592.3.115 by Robert Collins
Move same repository check up to Repository.fetch to allow all fetch implementations to benefit.
1098
        # fast path same-url fetch operations
1099
        if self.has_same_location(source):
1100
            # check that last_revision is in 'from' and then return a
1101
            # no-operation.
1102
            if (revision_id is not None and
1103
                not _mod_revision.is_null(revision_id)):
1104
                self.get_revision(revision_id)
1105
            return 0, []
3582.1.3 by Martin Pool
Repository.fetch no longer needs to translate NotImplementedErro to IncompatibleRepositories
1106
        # if there is no specific appropriate InterRepository, this will get
1107
        # the InterRepository base class, which raises an
1108
        # IncompatibleRepositories when asked to fetch.
2323.8.3 by Aaron Bentley
Reduce scope of try/except, update NEWS
1109
        inter = InterRepository.get(source, self)
3582.1.3 by Martin Pool
Repository.fetch no longer needs to translate NotImplementedErro to IncompatibleRepositories
1110
        return inter.fetch(revision_id=revision_id, pb=pb,
1111
            find_ghosts=find_ghosts)
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1112
2520.4.54 by Aaron Bentley
Hang a create_bundle method off repository
1113
    def create_bundle(self, target, base, fileobj, format=None):
1114
        return serializer.write_bundle(self, target, base, fileobj, format)
1115
2803.2.1 by Robert Collins
* CommitBuilder now advertises itself as requiring the root entry to be
1116
    def get_commit_builder(self, branch, parents, config, timestamp=None,
1117
                           timezone=None, committer=None, revprops=None,
1740.3.7 by Jelmer Vernooij
Move committer, log, revprops, timestamp and timezone to CommitBuilder.
1118
                           revision_id=None):
1119
        """Obtain a CommitBuilder for this repository.
1120
        
1121
        :param branch: Branch to commit to.
1122
        :param parents: Revision ids of the parents of the new revision.
1123
        :param config: Configuration to use.
1124
        :param timestamp: Optional timestamp recorded for commit.
1125
        :param timezone: Optional timezone for timestamp.
1126
        :param committer: Optional committer to set for commit.
1127
        :param revprops: Optional dictionary of revision properties.
1128
        :param revision_id: Optional revision id.
1129
        """
2818.3.2 by Robert Collins
Review feedback.
1130
        result = self._commit_builder_class(self, parents, config,
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
1131
            timestamp, timezone, committer, revprops, revision_id)
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1132
        self.start_write_group()
1133
        return result
1740.3.1 by Jelmer Vernooij
Introduce and use CommitBuilder objects.
1134
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1135
    def unlock(self):
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1136
        if (self.control_files._lock_count == 1 and
1137
            self.control_files._lock_mode == 'w'):
1138
            if self._write_group is not None:
2592.3.244 by Martin Pool
unlock while in a write group now aborts the write group, unlocks, and errors.
1139
                self.abort_write_group()
1140
                self.control_files.unlock()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1141
                raise errors.BzrError(
1142
                    'Must end write groups before releasing write locks.')
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1143
        self.control_files.unlock()
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
1144
        for repo in self._fallback_repositories:
1145
            repo.unlock()
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1146
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1147
    @needs_read_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1148
    def clone(self, a_bzrdir, revision_id=None):
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1149
        """Clone this repository into a_bzrdir using the current format.
1150
1151
        Currently no check is made that the format of this repository and
1152
        the bzrdir format are compatible. FIXME RBC 20060201.
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1153
1154
        :return: The newly created destination repository.
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1155
        """
2440.1.1 by Martin Pool
Add new Repository.sprout,
1156
        # TODO: deprecate after 0.16; cloning this with all its settings is
1157
        # probably not very useful -- mbp 20070423
1158
        dest_repo = self._create_sprouting_repo(a_bzrdir, shared=self.is_shared())
1159
        self.copy_content_into(dest_repo, revision_id)
1160
        return dest_repo
1161
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1162
    def start_write_group(self):
1163
        """Start a write group in the repository.
1164
1165
        Write groups are used by repositories which do not have a 1:1 mapping
1166
        between file ids and backend store to manage the insertion of data from
1167
        both fetch and commit operations.
1168
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1169
        A write lock is required around the start_write_group/commit_write_group
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1170
        for the support of lock-requiring repository formats.
2617.6.8 by Robert Collins
Review feedback and documentation.
1171
1172
        One can only insert data into a repository inside a write group.
1173
2617.6.6 by Robert Collins
Some review feedback.
1174
        :return: None.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1175
        """
2592.3.188 by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency.
1176
        if not self.is_write_locked():
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1177
            raise errors.NotWriteLocked(self)
1178
        if self._write_group:
1179
            raise errors.BzrError('already in a write group')
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1180
        self._start_write_group()
1181
        # so we can detect unlock/relock - the write group is now entered.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1182
        self._write_group = self.get_transaction()
1183
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1184
    def _start_write_group(self):
1185
        """Template method for per-repository write group startup.
1186
        
1187
        This is called before the write group is considered to be 
1188
        entered.
1189
        """
1190
2440.1.1 by Martin Pool
Add new Repository.sprout,
1191
    @needs_read_lock
1192
    def sprout(self, to_bzrdir, revision_id=None):
1193
        """Create a descendent repository for new development.
1194
1195
        Unlike clone, this does not copy the settings of the repository.
1196
        """
1197
        dest_repo = self._create_sprouting_repo(to_bzrdir, shared=False)
1198
        dest_repo.fetch(self, revision_id=revision_id)
1199
        return dest_repo
1200
1201
    def _create_sprouting_repo(self, a_bzrdir, shared):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1202
        if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):
1203
            # use target default format.
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1204
            dest_repo = a_bzrdir.create_repository()
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1205
        else:
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1206
            # Most control formats need the repository to be specifically
1207
            # created, but on some old all-in-one formats it's not needed
1208
            try:
2440.1.1 by Martin Pool
Add new Repository.sprout,
1209
                dest_repo = self._format.initialize(a_bzrdir, shared=shared)
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1210
            except errors.UninitializableFormat:
1211
                dest_repo = a_bzrdir.open_repository()
1212
        return dest_repo
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1213
1563.2.22 by Robert Collins
Move responsibility for repository.has_revision into RevisionStore
1214
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1215
    def has_revision(self, revision_id):
1563.2.22 by Robert Collins
Move responsibility for repository.has_revision into RevisionStore
1216
        """True if this repository has a copy of the revision."""
3172.3.1 by Robert Collins
Repository has a new method ``has_revisions`` which signals the presence
1217
        return revision_id in self.has_revisions((revision_id,))
1218
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1219
    @needs_read_lock
3172.3.1 by Robert Collins
Repository has a new method ``has_revisions`` which signals the presence
1220
    def has_revisions(self, revision_ids):
1221
        """Probe to find out the presence of multiple revisions.
1222
1223
        :param revision_ids: An iterable of revision_ids.
1224
        :return: A set of the revision_ids that were present.
1225
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1226
        parent_map = self.revisions.get_parent_map(
1227
            [(rev_id,) for rev_id in revision_ids])
1228
        result = set()
1229
        if _mod_revision.NULL_REVISION in revision_ids:
1230
            result.add(_mod_revision.NULL_REVISION)
1231
        result.update([key[0] for key in parent_map])
1232
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1233
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1234
    @needs_read_lock
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1235
    def get_revision(self, revision_id):
1236
        """Return the Revision object for a named revision."""
1237
        return self.get_revisions([revision_id])[0]
1238
1239
    @needs_read_lock
1570.1.13 by Robert Collins
Check for incorrect revision parentage in the weave during revision access.
1240
    def get_revision_reconcile(self, revision_id):
1241
        """'reconcile' helper routine that allows access to a revision always.
1242
        
1243
        This variant of get_revision does not cross check the weave graph
1244
        against the revision one as get_revision does: but it should only
1245
        be used by reconcile, or reconcile-alike commands that are correcting
1246
        or testing the revision graph.
1247
        """
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1248
        return self._get_revisions([revision_id])[0]
2249.5.13 by John Arbash Meinel
Finish auditing Repository, and fix generate_ids to always generate utf8 ids.
1249
1756.1.2 by Aaron Bentley
Show logs using get_revisions
1250
    @needs_read_lock
1251
    def get_revisions(self, revision_ids):
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1252
        """Get many revisions at once."""
1253
        return self._get_revisions(revision_ids)
1254
1255
    @needs_read_lock
1256
    def _get_revisions(self, revision_ids):
1257
        """Core work logic to get many revisions without sanity checks."""
1258
        for rev_id in revision_ids:
1259
            if not rev_id or not isinstance(rev_id, basestring):
1260
                raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1261
        keys = [(key,) for key in revision_ids]
1262
        stream = self.revisions.get_record_stream(keys, 'unordered', True)
1263
        revs = {}
1264
        for record in stream:
1265
            if record.storage_kind == 'absent':
1266
                raise errors.NoSuchRevision(self, record.key[0])
1267
            text = record.get_bytes_as('fulltext')
1268
            rev = self._serializer.read_revision_from_string(text)
1269
            revs[record.key[0]] = rev
1270
        return [revs[revid] for revid in revision_ids]
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1271
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1272
    @needs_read_lock
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1273
    def get_revision_xml(self, revision_id):
2249.5.13 by John Arbash Meinel
Finish auditing Repository, and fix generate_ids to always generate utf8 ids.
1274
        # TODO: jam 20070210 This shouldn't be necessary since get_revision
1275
        #       would have already do it.
1276
        # TODO: jam 20070210 Just use _serializer.write_revision_to_string()
1277
        rev = self.get_revision(revision_id)
3224.5.25 by Andrew Bennetts
More minor import tidying suggested by pyflakes.
1278
        rev_tmp = cStringIO.StringIO()
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1279
        # the current serializer..
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1280
        self._serializer.write_revision(rev, rev_tmp)
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1281
        rev_tmp.seek(0)
1282
        return rev_tmp.getvalue()
1283
1756.3.22 by Aaron Bentley
Tweaks from review
1284
    def get_deltas_for_revisions(self, revisions):
1756.3.19 by Aaron Bentley
Documentation and cleanups
1285
        """Produce a generator of revision deltas.
1286
        
1287
        Note that the input is a sequence of REVISIONS, not revision_ids.
1288
        Trees will be held in memory until the generator exits.
1289
        Each delta is relative to the revision's lefthand predecessor.
1290
        """
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1291
        required_trees = set()
1292
        for revision in revisions:
1293
            required_trees.add(revision.revision_id)
1294
            required_trees.update(revision.parent_ids[:1])
1295
        trees = dict((t.get_revision_id(), t) for 
1296
                     t in self.revision_trees(required_trees))
1297
        for revision in revisions:
1298
            if not revision.parent_ids:
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
1299
                old_tree = self.revision_tree(_mod_revision.NULL_REVISION)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1300
            else:
1301
                old_tree = trees[revision.parent_ids[0]]
1852.10.3 by Robert Collins
Remove all uses of compare_trees and replace with Tree.changes_from throughout bzrlib.
1302
            yield trees[revision.revision_id].changes_from(old_tree)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1303
1756.3.19 by Aaron Bentley
Documentation and cleanups
1304
    @needs_read_lock
1744.2.2 by Johan Rydberg
Add get_revision_delta to Repository; and make Branch.get_revision_delta use it.
1305
    def get_revision_delta(self, revision_id):
1306
        """Return the delta for one revision.
1307
1308
        The delta is relative to the left-hand predecessor of the
1309
        revision.
1310
        """
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1311
        r = self.get_revision(revision_id)
1756.3.22 by Aaron Bentley
Tweaks from review
1312
        return list(self.get_deltas_for_revisions([r]))[0]
1744.2.2 by Johan Rydberg
Add get_revision_delta to Repository; and make Branch.get_revision_delta use it.
1313
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1314
    @needs_write_lock
1315
    def store_revision_signature(self, gpg_strategy, plaintext, revision_id):
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1316
        signature = gpg_strategy.sign(plaintext)
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
1317
        self.add_signature_text(revision_id, signature)
2996.2.3 by Aaron Bentley
Add tests for install_revisions and add_signature
1318
1319
    @needs_write_lock
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
1320
    def add_signature_text(self, revision_id, signature):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1321
        self.signatures.add_lines((revision_id,), (),
1322
            osutils.split_lines(signature))
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1323
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1324
    def find_text_key_references(self):
1325
        """Find the text key references within the repository.
1326
1327
        :return: a dictionary mapping (file_id, revision_id) tuples to altered file-ids to an iterable of
1328
        revision_ids. Each altered file-ids has the exact revision_ids that
1329
        altered it listed explicitly.
1330
        :return: A dictionary mapping text keys ((fileid, revision_id) tuples)
1331
            to whether they were referred to by the inventory of the
1332
            revision_id that they contain. The inventory texts from all present
1333
            revision ids are assessed to generate this report.
1334
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1335
        revision_keys = self.revisions.keys()
1336
        w = self.inventories
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1337
        pb = ui.ui_factory.nested_progress_bar()
1338
        try:
1339
            return self._find_text_key_references_from_xml_inventory_lines(
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1340
                w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1341
        finally:
1342
            pb.finished()
1343
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1344
    def _find_text_key_references_from_xml_inventory_lines(self,
1345
        line_iterator):
1346
        """Core routine for extracting references to texts from inventories.
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1347
1348
        This performs the translation of xml lines to revision ids.
1349
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
1350
        :param line_iterator: An iterator of lines, origin_version_id
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1351
        :return: A dictionary mapping text keys ((fileid, revision_id) tuples)
1352
            to whether they were referred to by the inventory of the
1353
            revision_id that they contain. Note that if that revision_id was
1354
            not part of the line_iterator's output then False will be given -
1355
            even though it may actually refer to that key.
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1356
        """
2988.2.2 by Robert Collins
Review feedback.
1357
        if not self._serializer.support_altered_by_hack:
1358
            raise AssertionError(
1359
                "_find_text_key_references_from_xml_inventory_lines only "
1360
                "supported for branches which store inventory as unnested xml"
1361
                ", not on %r" % self)
1694.2.6 by Martin Pool
[merge] bzr.dev
1362
        result = {}
1563.2.35 by Robert Collins
cleanup deprecation warnings and finish conversion so the inventory is knit based too.
1363
1694.2.6 by Martin Pool
[merge] bzr.dev
1364
        # this code needs to read every new line in every inventory for the
1365
        # inventories [revision_ids]. Seeing a line twice is ok. Seeing a line
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
1366
        # not present in one of those inventories is unnecessary but not 
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
1367
        # harmful because we are filtering by the revision id marker in the
1694.2.6 by Martin Pool
[merge] bzr.dev
1368
        # inventory lines : we only select file ids altered in one of those  
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
1369
        # revisions. We don't need to see all lines in the inventory because
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
1370
        # only those added in an inventory in rev X can contain a revision=X
1371
        # line.
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1372
        unescape_revid_cache = {}
1373
        unescape_fileid_cache = {}
1374
2163.2.5 by John Arbash Meinel
Inline the cache lookup, and explain why
1375
        # jam 20061218 In a big fetch, this handles hundreds of thousands
1376
        # of lines, so it has had a lot of inlining and optimizing done.
1377
        # Sorry that it is a little bit messy.
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1378
        # Move several functions to be local variables, since this is a long
1379
        # running loop.
1380
        search = self._file_ids_altered_regex.search
2163.2.5 by John Arbash Meinel
Inline the cache lookup, and explain why
1381
        unescape = _unescape_xml
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1382
        setdefault = result.setdefault
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1383
        for line, line_key in line_iterator:
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1384
            match = search(line)
1385
            if match is None:
1386
                continue
1387
            # One call to match.group() returning multiple items is quite a
1388
            # bit faster than 2 calls to match.group() each returning 1
1389
            file_id, revision_id = match.group('file_id', 'revision_id')
1390
1391
            # Inlining the cache lookups helps a lot when you make 170,000
1392
            # lines and 350k ids, versus 8.4 unique ids.
1393
            # Using a cache helps in 2 ways:
1394
            #   1) Avoids unnecessary decoding calls
1395
            #   2) Re-uses cached strings, which helps in future set and
1396
            #      equality checks.
1397
            # (2) is enough that removing encoding entirely along with
1398
            # the cache (so we are using plain strings) results in no
1399
            # performance improvement.
1400
            try:
1401
                revision_id = unescape_revid_cache[revision_id]
1402
            except KeyError:
1403
                unescaped = unescape(revision_id)
1404
                unescape_revid_cache[revision_id] = unescaped
1405
                revision_id = unescaped
1406
2988.2.2 by Robert Collins
Review feedback.
1407
            # Note that unconditionally unescaping means that we deserialise
1408
            # every fileid, which for general 'pull' is not great, but we don't
1409
            # really want to have some many fulltexts that this matters anyway.
1410
            # RBC 20071114.
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1411
            try:
1412
                file_id = unescape_fileid_cache[file_id]
1413
            except KeyError:
1414
                unescaped = unescape(file_id)
1415
                unescape_fileid_cache[file_id] = unescaped
1416
                file_id = unescaped
1417
1418
            key = (file_id, revision_id)
1419
            setdefault(key, False)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1420
            if revision_id == line_key[-1]:
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1421
                result[key] = True
1422
        return result
1423
1424
    def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
1425
        revision_ids):
1426
        """Helper routine for fileids_altered_by_revision_ids.
1427
1428
        This performs the translation of xml lines to revision ids.
1429
1430
        :param line_iterator: An iterator of lines, origin_version_id
1431
        :param revision_ids: The revision ids to filter for. This should be a
1432
            set or other type which supports efficient __contains__ lookups, as
1433
            the revision id from each parsed line will be looked up in the
1434
            revision_ids filter.
1435
        :return: a dictionary mapping altered file-ids to an iterable of
1436
        revision_ids. Each altered file-ids has the exact revision_ids that
1437
        altered it listed explicitly.
1438
        """
1439
        result = {}
1440
        setdefault = result.setdefault
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1441
        for key in \
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1442
            self._find_text_key_references_from_xml_inventory_lines(
1443
                line_iterator).iterkeys():
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
1444
            # once data is all ensured-consistent; then this is
1445
            # if revision_id == version_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1446
            if key[-1:] in revision_ids:
1447
                setdefault(key[0], set()).add(key[-1])
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1448
        return result
1449
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
1450
    def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1451
        """Find the file ids and versions affected by revisions.
1452
1453
        :param revisions: an iterable containing revision ids.
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
1454
        :param _inv_weave: The inventory weave from this repository or None.
1455
            If None, the inventory weave will be opened automatically.
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1456
        :return: a dictionary mapping altered file-ids to an iterable of
1457
        revision_ids. Each altered file-ids has the exact revision_ids that
1458
        altered it listed explicitly.
1459
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1460
        selected_keys = set((revid,) for revid in revision_ids)
1461
        w = _inv_weave or self.inventories
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
1462
        pb = ui.ui_factory.nested_progress_bar()
1463
        try:
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1464
            return self._find_file_ids_from_xml_inventory_lines(
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1465
                w.iter_lines_added_or_present_in_keys(
1466
                    selected_keys, pb=pb),
1467
                selected_keys)
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
1468
        finally:
1469
            pb.finished()
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1470
2708.1.7 by Aaron Bentley
Rename extract_files_bytes to iter_files_bytes
1471
    def iter_files_bytes(self, desired_files):
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1472
        """Iterate through file versions.
1473
2708.1.10 by Aaron Bentley
Update docstrings
1474
        Files will not necessarily be returned in the order they occur in
1475
        desired_files.  No specific order is guaranteed.
1476
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1477
        Yields pairs of identifier, bytes_iterator.  identifier is an opaque
2708.1.10 by Aaron Bentley
Update docstrings
1478
        value supplied by the caller as part of desired_files.  It should
1479
        uniquely identify the file version in the caller's context.  (Examples:
1480
        an index number or a TreeTransform trans_id.)
1481
1482
        bytes_iterator is an iterable of bytestrings for the file.  The
1483
        kind of iterable and length of the bytestrings are unspecified, but for
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1484
        this implementation, it is a list of bytes produced by
1485
        VersionedFile.get_record_stream().
2708.1.10 by Aaron Bentley
Update docstrings
1486
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1487
        :param desired_files: a list of (file_id, revision_id, identifier)
2708.1.10 by Aaron Bentley
Update docstrings
1488
            triples
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1489
        """
2708.1.3 by Aaron Bentley
Implement extract_files_bytes on Repository
1490
        transaction = self.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1491
        text_keys = {}
2708.1.3 by Aaron Bentley
Implement extract_files_bytes on Repository
1492
        for file_id, revision_id, callable_data in desired_files:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1493
            text_keys[(file_id, revision_id)] = callable_data
1494
        for record in self.texts.get_record_stream(text_keys, 'unordered', True):
1495
            if record.storage_kind == 'absent':
1496
                raise errors.RevisionNotPresent(record.key, self)
1497
            yield text_keys[record.key], record.get_bytes_as('fulltext')
2708.1.3 by Aaron Bentley
Implement extract_files_bytes on Repository
1498
3063.2.1 by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted.
1499
    def _generate_text_key_index(self, text_key_references=None,
1500
        ancestors=None):
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1501
        """Generate a new text key index for the repository.
1502
1503
        This is an expensive function that will take considerable time to run.
1504
1505
        :return: A dict mapping text keys ((file_id, revision_id) tuples) to a
1506
            list of parents, also text keys. When a given key has no parents,
1507
            the parents list will be [NULL_REVISION].
1508
        """
1509
        # All revisions, to find inventory parents.
3063.2.1 by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted.
1510
        if ancestors is None:
3287.6.1 by Robert Collins
* ``VersionedFile.get_graph`` is deprecated, with no replacement method.
1511
            graph = self.get_graph()
1512
            ancestors = graph.get_parent_map(self.all_revision_ids())
2951.2.9 by Robert Collins
* ``pack-0.92`` repositories can now be reconciled.
1513
        if text_key_references is None:
1514
            text_key_references = self.find_text_key_references()
2988.3.1 by Robert Collins
Handle the progress bar in _generate_text_key_index correctly.
1515
        pb = ui.ui_factory.nested_progress_bar()
1516
        try:
1517
            return self._do_generate_text_key_index(ancestors,
1518
                text_key_references, pb)
1519
        finally:
1520
            pb.finished()
1521
1522
    def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
1523
        """Helper for _generate_text_key_index to avoid deep nesting."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1524
        revision_order = tsort.topo_sort(ancestors)
1525
        invalid_keys = set()
1526
        revision_keys = {}
1527
        for revision_id in revision_order:
1528
            revision_keys[revision_id] = set()
1529
        text_count = len(text_key_references)
1530
        # a cache of the text keys to allow reuse; costs a dict of all the
1531
        # keys, but saves a 2-tuple for every child of a given key.
1532
        text_key_cache = {}
1533
        for text_key, valid in text_key_references.iteritems():
1534
            if not valid:
1535
                invalid_keys.add(text_key)
1536
            else:
1537
                revision_keys[text_key[1]].add(text_key)
1538
            text_key_cache[text_key] = text_key
1539
        del text_key_references
1540
        text_index = {}
1541
        text_graph = graph.Graph(graph.DictParentsProvider(text_index))
1542
        NULL_REVISION = _mod_revision.NULL_REVISION
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
1543
        # Set a cache with a size of 10 - this suffices for bzr.dev but may be
1544
        # too small for large or very branchy trees. However, for 55K path
1545
        # trees, it would be easy to use too much memory trivially. Ideally we
1546
        # could gauge this by looking at available real memory etc, but this is
1547
        # always a tricky proposition.
1548
        inventory_cache = lru_cache.LRUCache(10)
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1549
        batch_size = 10 # should be ~150MB on a 55K path tree
1550
        batch_count = len(revision_order) / batch_size + 1
1551
        processed_texts = 0
1552
        pb.update("Calculating text parents.", processed_texts, text_count)
1553
        for offset in xrange(batch_count):
1554
            to_query = revision_order[offset * batch_size:(offset + 1) *
1555
                batch_size]
1556
            if not to_query:
1557
                break
1558
            for rev_tree in self.revision_trees(to_query):
1559
                revision_id = rev_tree.get_revision_id()
1560
                parent_ids = ancestors[revision_id]
1561
                for text_key in revision_keys[revision_id]:
1562
                    pb.update("Calculating text parents.", processed_texts)
1563
                    processed_texts += 1
1564
                    candidate_parents = []
1565
                    for parent_id in parent_ids:
1566
                        parent_text_key = (text_key[0], parent_id)
1567
                        try:
1568
                            check_parent = parent_text_key not in \
1569
                                revision_keys[parent_id]
1570
                        except KeyError:
1571
                            # the parent parent_id is a ghost:
1572
                            check_parent = False
1573
                            # truncate the derived graph against this ghost.
1574
                            parent_text_key = None
1575
                        if check_parent:
1576
                            # look at the parent commit details inventories to
1577
                            # determine possible candidates in the per file graph.
1578
                            # TODO: cache here.
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
1579
                            try:
1580
                                inv = inventory_cache[parent_id]
1581
                            except KeyError:
1582
                                inv = self.revision_tree(parent_id).inventory
1583
                                inventory_cache[parent_id] = inv
1584
                            parent_entry = inv._byid.get(text_key[0], None)
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1585
                            if parent_entry is not None:
1586
                                parent_text_key = (
1587
                                    text_key[0], parent_entry.revision)
1588
                            else:
1589
                                parent_text_key = None
1590
                        if parent_text_key is not None:
1591
                            candidate_parents.append(
1592
                                text_key_cache[parent_text_key])
1593
                    parent_heads = text_graph.heads(candidate_parents)
1594
                    new_parents = list(parent_heads)
1595
                    new_parents.sort(key=lambda x:candidate_parents.index(x))
1596
                    if new_parents == []:
1597
                        new_parents = [NULL_REVISION]
1598
                    text_index[text_key] = new_parents
1599
1600
        for text_key in invalid_keys:
1601
            text_index[text_key] = [NULL_REVISION]
1602
        return text_index
1603
2668.2.8 by Andrew Bennetts
Rename get_data_to_fetch_for_revision_ids as item_keys_introduced_by.
1604
    def item_keys_introduced_by(self, revision_ids, _files_pb=None):
1605
        """Get an iterable listing the keys of all the data introduced by a set
1606
        of revision IDs.
1607
1608
        The keys will be ordered so that the corresponding items can be safely
1609
        fetched and inserted in that order.
1610
1611
        :returns: An iterable producing tuples of (knit-kind, file-id,
1612
            versions).  knit-kind is one of 'file', 'inventory', 'signatures',
1613
            'revisions'.  file-id is None unless knit-kind is 'file'.
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1614
        """
1615
        # XXX: it's a bit weird to control the inventory weave caching in this
2535.3.7 by Andrew Bennetts
Remove now unused _fetch_weave_texts, make progress reporting closer to how it was before I refactored __fetch.
1616
        # generator.  Ideally the caching would be done in fetch.py I think.  Or
1617
        # maybe this generator should explicitly have the contract that it
1618
        # should not be iterated until the previously yielded item has been
1619
        # processed?
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1620
        inv_w = self.inventories
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1621
1622
        # file ids that changed
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
1623
        file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
2535.3.8 by Andrew Bennetts
Unbreak progress reporting.
1624
        count = 0
1625
        num_file_ids = len(file_ids)
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1626
        for file_id, altered_versions in file_ids.iteritems():
2668.2.8 by Andrew Bennetts
Rename get_data_to_fetch_for_revision_ids as item_keys_introduced_by.
1627
            if _files_pb is not None:
1628
                _files_pb.update("fetch texts", count, num_file_ids)
2535.3.8 by Andrew Bennetts
Unbreak progress reporting.
1629
            count += 1
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1630
            yield ("file", file_id, altered_versions)
2535.3.9 by Andrew Bennetts
More comments.
1631
        # We're done with the files_pb.  Note that it finished by the caller,
1632
        # just as it was created by the caller.
2668.2.8 by Andrew Bennetts
Rename get_data_to_fetch_for_revision_ids as item_keys_introduced_by.
1633
        del _files_pb
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1634
1635
        # inventory
1636
        yield ("inventory", None, revision_ids)
1637
1638
        # signatures
3825.5.2 by Andrew Bennetts
Ensure that item_keys_introduced_by returns the
1639
        # XXX: Note ATM no callers actually pay attention to this return
1640
        #      instead they just use the list of revision ids and ignore
1641
        #      missing sigs. Consider removing this work entirely
1642
        revisions_with_signatures = set(self.signatures.get_parent_map(
1643
            [(r,) for r in revision_ids]))
3825.5.1 by Andrew Bennetts
Improve determining signatures to transfer in item_keys_introduced_by.
1644
        revisions_with_signatures = set(
3825.5.2 by Andrew Bennetts
Ensure that item_keys_introduced_by returns the
1645
            [r for (r,) in revisions_with_signatures])
3825.5.1 by Andrew Bennetts
Improve determining signatures to transfer in item_keys_introduced_by.
1646
        revisions_with_signatures.intersection_update(revision_ids)
2535.3.25 by Andrew Bennetts
Fetch signatures too.
1647
        yield ("signatures", None, revisions_with_signatures)
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
1648
1649
        # revisions
1650
        yield ("revisions", None, revision_ids)
1651
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1652
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1653
    def get_inventory(self, revision_id):
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
1654
        """Get Inventory object by revision id."""
1655
        return self.iter_inventories([revision_id]).next()
1656
1657
    def iter_inventories(self, revision_ids):
1658
        """Get many inventories by revision_ids.
1659
1660
        This will buffer some or all of the texts used in constructing the
1661
        inventories in memory, but will only parse a single inventory at a
1662
        time.
1663
1664
        :return: An iterator of inventories.
1665
        """
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1666
        if ((None in revision_ids)
1667
            or (_mod_revision.NULL_REVISION in revision_ids)):
1668
            raise ValueError('cannot get null revision inventory')
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
1669
        return self._iter_inventories(revision_ids)
1670
1671
    def _iter_inventories(self, revision_ids):
1672
        """single-document based inventory iteration."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1673
        for text, revision_id in self._iter_inventory_xmls(revision_ids):
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
1674
            yield self.deserialise_inventory(revision_id, text)
1740.2.3 by Aaron Bentley
Only reserialize the working tree basis inventory when needed.
1675
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1676
    def _iter_inventory_xmls(self, revision_ids):
1677
        keys = [(revision_id,) for revision_id in revision_ids]
1678
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
1679
        texts = {}
1680
        for record in stream:
1681
            if record.storage_kind != 'absent':
1682
                texts[record.key] = record.get_bytes_as('fulltext')
1683
            else:
1684
                raise errors.NoSuchRevision(self, record.key)
1685
        for key in keys:
1686
            yield texts[key], key[-1]
1687
1740.2.3 by Aaron Bentley
Only reserialize the working tree basis inventory when needed.
1688
    def deserialise_inventory(self, revision_id, xml):
1689
        """Transform the xml into an inventory object. 
1690
1691
        :param revision_id: The expected revision id of the inventory.
1692
        :param xml: A serialised inventory.
1693
        """
3169.2.2 by Robert Collins
Add a test to Repository.deserialise_inventory that the resulting ivnentory is the one asked for, and update relevant tests. Also tweak the model 1 to 2 regenerate inventories logic to use the revision trees parent marker which is more accurate in some cases.
1694
        result = self._serializer.read_inventory_from_string(xml, revision_id)
3169.2.3 by Robert Collins
Use an if, not an assert, as we test with -O.
1695
        if result.revision_id != revision_id:
1696
            raise AssertionError('revision id mismatch %s != %s' % (
1697
                result.revision_id, revision_id))
3169.2.2 by Robert Collins
Add a test to Repository.deserialise_inventory that the resulting ivnentory is the one asked for, and update relevant tests. Also tweak the model 1 to 2 regenerate inventories logic to use the revision trees parent marker which is more accurate in some cases.
1698
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1699
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
1700
    def serialise_inventory(self, inv):
1910.2.48 by Aaron Bentley
Update from review comments
1701
        return self._serializer.write_inventory_to_string(inv)
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
1702
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
1703
    def _serialise_inventory_to_lines(self, inv):
1704
        return self._serializer.write_inventory_to_lines(inv)
1705
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
1706
    def get_serializer_format(self):
1707
        return self._serializer.format_num
1708
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1709
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1710
    def get_inventory_xml(self, revision_id):
1711
        """Get inventory XML as a file object."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1712
        texts = self._iter_inventory_xmls([revision_id])
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1713
        try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1714
            text, revision_id = texts.next()
1715
        except StopIteration:
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1716
            raise errors.HistoryMissing(self, 'inventory', revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1717
        return text
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1718
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1719
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1720
    def get_inventory_sha1(self, revision_id):
1721
        """Return the sha1 hash of the inventory entry
1722
        """
1723
        return self.get_revision(revision_id).inventory_sha1
1724
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
1725
    def iter_reverse_revision_history(self, revision_id):
1726
        """Iterate backwards through revision ids in the lefthand history
1727
1728
        :param revision_id: The revision id to start with.  All its lefthand
1729
            ancestors will be traversed.
1730
        """
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1731
        graph = self.get_graph()
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
1732
        next_id = revision_id
1733
        while True:
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1734
            if next_id in (None, _mod_revision.NULL_REVISION):
1735
                return
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
1736
            yield next_id
3287.5.10 by Robert Collins
Note iter_reverse_revision_history exception decision.
1737
            # Note: The following line may raise KeyError in the event of
1738
            # truncated history. We decided not to have a try:except:raise
1739
            # RevisionNotPresent here until we see a use for it, because of the
1740
            # cost in an inner loop that is by its very nature O(history).
1741
            # Robert Collins 20080326
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1742
            parents = graph.get_parent_map([next_id])[next_id]
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
1743
            if len(parents) == 0:
1744
                return
1745
            else:
1746
                next_id = parents[0]
1747
1594.2.3 by Robert Collins
bugfix revision.MultipleRevisionSources.get_revision_graph to integrate ghosts between sources. [slow on weaves, fast on knits.
1748
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1749
    def get_revision_inventory(self, revision_id):
1750
        """Return inventory of a past revision."""
1751
        # TODO: Unify this with get_inventory()
1752
        # bzr 0.0.6 and later imposes the constraint that the inventory_id
1753
        # must be the same as its revision, so this is trivial.
1534.4.28 by Robert Collins
first cut at merge from integration.
1754
        if revision_id is None:
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1755
            # This does not make sense: if there is no revision,
1756
            # then it is the current tree inventory surely ?!
1757
            # and thus get_root_id() is something that looks at the last
1758
            # commit on the branch, and the get_root_id is an inventory check.
1759
            raise NotImplementedError
1760
            # return Inventory(self.get_root_id())
1761
        else:
1762
            return self.get_inventory(revision_id)
1763
1534.6.3 by Robert Collins
find_repository sufficiently robust.
1764
    def is_shared(self):
1765
        """Return True if this repository is flagged as a shared repository."""
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
1766
        raise NotImplementedError(self.is_shared)
1534.6.3 by Robert Collins
find_repository sufficiently robust.
1767
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
1768
    @needs_write_lock
1692.1.1 by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow
1769
    def reconcile(self, other=None, thorough=False):
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
1770
        """Reconcile this repository."""
1771
        from bzrlib.reconcile import RepoReconciler
1692.1.1 by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow
1772
        reconciler = RepoReconciler(self, thorough=thorough)
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
1773
        reconciler.reconcile()
1774
        return reconciler
2440.1.1 by Martin Pool
Add new Repository.sprout,
1775
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1776
    def _refresh_data(self):
1777
        """Helper called from lock_* to ensure coherency with disk.
1778
1779
        The default implementation does nothing; it is however possible
1780
        for repositories to maintain loaded indices across multiple locks
1781
        by checking inside their implementation of this method to see
1782
        whether their indices are still valid. This depends of course on
1783
        the disk format being validatable in this manner.
1784
        """
1785
1534.6.3 by Robert Collins
find_repository sufficiently robust.
1786
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1787
    def revision_tree(self, revision_id):
1788
        """Return Tree for a revision on this branch.
1789
3668.5.2 by Jelmer Vernooij
Fix docstring.
1790
        `revision_id` may be NULL_REVISION for the empty tree revision.
1852.5.1 by Robert Collins
Deprecate EmptyTree in favour of using Repository.revision_tree.
1791
        """
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
1792
        revision_id = _mod_revision.ensure_null(revision_id)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1793
        # TODO: refactor this to use an existing revision object
1794
        # so we don't need to read it in twice.
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
1795
        if revision_id == _mod_revision.NULL_REVISION:
1731.1.61 by Aaron Bentley
Merge bzr.dev
1796
            return RevisionTree(self, Inventory(root_id=None), 
1797
                                _mod_revision.NULL_REVISION)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1798
        else:
1799
            inv = self.get_revision_inventory(revision_id)
1185.65.17 by Robert Collins
Merge from integration, mode-changes are broken.
1800
            return RevisionTree(self, inv, revision_id)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1801
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1802
    def revision_trees(self, revision_ids):
1803
        """Return Tree for a revision on this branch.
1804
1756.3.19 by Aaron Bentley
Documentation and cleanups
1805
        `revision_id` may not be None or 'null:'"""
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
1806
        inventories = self.iter_inventories(revision_ids)
1807
        for inv in inventories:
1808
            yield RevisionTree(self, inv, inv.revision_id)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1809
1810
    @needs_read_lock
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
1811
    def get_ancestry(self, revision_id, topo_sorted=True):
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
1812
        """Return a list of revision-ids integrated by a revision.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1813
1814
        The first element of the list is always None, indicating the origin 
1815
        revision.  This might change when we have history horizons, or 
1816
        perhaps we should have a new API.
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
1817
        
1818
        This is topologically sorted.
1819
        """
2598.5.1 by Aaron Bentley
Start eliminating the use of None to indicate null revision
1820
        if _mod_revision.is_null(revision_id):
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
1821
            return [None]
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1822
        if not self.has_revision(revision_id):
1823
            raise errors.NoSuchRevision(self, revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1824
        graph = self.get_graph()
1825
        keys = set()
1826
        search = graph._make_breadth_first_searcher([revision_id])
1827
        while True:
1828
            try:
1829
                found, ghosts = search.next_with_ghosts()
1830
            except StopIteration:
1831
                break
1832
            keys.update(found)
1833
        if _mod_revision.NULL_REVISION in keys:
1834
            keys.remove(_mod_revision.NULL_REVISION)
1835
        if topo_sorted:
1836
            parent_map = graph.get_parent_map(keys)
1837
            keys = tsort.topo_sort(parent_map)
1838
        return [None] + list(keys)
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
1839
2604.2.1 by Robert Collins
(robertc) Introduce a pack command.
1840
    def pack(self):
1841
        """Compress the data within the repository.
1842
1843
        This operation only makes sense for some repository types. For other
1844
        types it should be a no-op that just returns.
1845
1846
        This stub method does not require a lock, but subclasses should use
1847
        @needs_write_lock as this is a long running call its reasonable to 
1848
        implicitly lock for the user.
1849
        """
1850
1185.65.4 by Aaron Bentley
Fixed cat command
1851
    @needs_read_lock
3511.1.2 by Aaron Bentley
Update text and deprecation symbols.
1852
    @deprecated_method(one_six)
1185.65.4 by Aaron Bentley
Fixed cat command
1853
    def print_file(self, file, revision_id):
1185.65.29 by Robert Collins
Implement final review suggestions.
1854
        """Print `file` to stdout.
1855
        
1856
        FIXME RBC 20060125 as John Meinel points out this is a bad api
1857
        - it writes to stdout, it assumes that that is valid etc. Fix
1858
        by creating a new more flexible convenience function.
1859
        """
1185.65.4 by Aaron Bentley
Fixed cat command
1860
        tree = self.revision_tree(revision_id)
1861
        # use inventory as it was in that revision
1862
        file_id = tree.inventory.path2id(file)
1863
        if not file_id:
1685.1.26 by John Arbash Meinel
Repository had a bug with what exception was raised when a file was missing
1864
            # TODO: jam 20060427 Write a test for this code path
1865
            #       it had a bug in it, and was raising the wrong
1866
            #       exception.
1867
            raise errors.BzrError("%r is not present in revision %s" % (file, revision_id))
1185.65.4 by Aaron Bentley
Fixed cat command
1868
        tree.print_file(file_id)
1869
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1870
    def get_transaction(self):
1871
        return self.control_files.get_transaction()
1872
3341.2.2 by Alexander Belchenko
Tree.print_file and Repository.print_file are deprecated.
1873
    @deprecated_method(one_one)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
1874
    def get_parents(self, revision_ids):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1875
        """See StackedParentsProvider.get_parents"""
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1876
        parent_map = self.get_parent_map(revision_ids)
1877
        return [parent_map.get(r, None) for r in revision_ids]
1878
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
1879
    def get_parent_map(self, revision_ids):
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1880
        """See graph._StackedParentsProvider.get_parent_map"""
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
1881
        # revisions index works in keys; this just works in revisions
1882
        # therefore wrap and unwrap
1883
        query_keys = []
1884
        result = {}
1885
        for revision_id in revision_ids:
1886
            if revision_id == _mod_revision.NULL_REVISION:
1887
                result[revision_id] = ()
1888
            elif revision_id is None:
3373.5.2 by John Arbash Meinel
Add repository_implementation tests for get_parent_map
1889
                raise ValueError('get_parent_map(None) is not valid')
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
1890
            else:
1891
                query_keys.append((revision_id ,))
1892
        for ((revision_id,), parent_keys) in \
1893
                self.revisions.get_parent_map(query_keys).iteritems():
1894
            if parent_keys:
1895
                result[revision_id] = tuple(parent_revid
1896
                    for (parent_revid,) in parent_keys)
1897
            else:
1898
                result[revision_id] = (_mod_revision.NULL_REVISION,)
1899
        return result
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
1900
1901
    def _make_parents_provider(self):
1902
        return self
1903
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
1904
    def get_graph(self, other_repository=None):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
1905
        """Return the graph walker for this repository format"""
1906
        parents_provider = self._make_parents_provider()
2490.2.14 by Aaron Bentley
Avoid StackedParentsProvider when underlying repos match
1907
        if (other_repository is not None and
3211.3.1 by Jelmer Vernooij
Use convenience function to check whether two repository handles are referring to the same repository.
1908
            not self.has_same_location(other_repository)):
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
1909
            parents_provider = graph._StackedParentsProvider(
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
1910
                [parents_provider, other_repository._make_parents_provider()])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1911
        return graph.Graph(parents_provider)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
1912
3036.1.3 by Robert Collins
Privatise VersionedFileChecker.
1913
    def _get_versioned_file_checker(self):
2988.1.6 by Robert Collins
Change the contract for VersionedFileChecker to consolidate related parameters rather than splitting them across two api calls. This allows better reuse of a single checker object.
1914
        """Return an object suitable for checking versioned files."""
3036.1.3 by Robert Collins
Privatise VersionedFileChecker.
1915
        return _VersionedFileChecker(self)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
1916
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
1917
    def revision_ids_to_search_result(self, result_set):
1918
        """Convert a set of revision ids to a graph SearchResult."""
1919
        result_parents = set()
1920
        for parents in self.get_graph().get_parent_map(
1921
            result_set).itervalues():
1922
            result_parents.update(parents)
1923
        included_keys = result_set.intersection(result_parents)
1924
        start_keys = result_set.difference(included_keys)
1925
        exclude_keys = result_parents.difference(result_set)
1926
        result = graph.SearchResult(start_keys, exclude_keys,
1927
            len(result_set), result_set)
1928
        return result
1929
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1930
    @needs_write_lock
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
1931
    def set_make_working_trees(self, new_value):
1932
        """Set the policy flag for making working trees when creating branches.
1933
1934
        This only applies to branches that use this repository.
1935
1936
        The default is 'True'.
1937
        :param new_value: True to restore the default, False to disable making
1938
                          working trees.
1939
        """
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
1940
        raise NotImplementedError(self.set_make_working_trees)
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
1941
    
1942
    def make_working_trees(self):
1943
        """Returns the policy for making working trees on new branches."""
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
1944
        raise NotImplementedError(self.make_working_trees)
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
1945
1946
    @needs_write_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1947
    def sign_revision(self, revision_id, gpg_strategy):
1948
        plaintext = Testament.from_revision(self, revision_id).as_short_text()
1949
        self.store_revision_signature(gpg_strategy, plaintext, revision_id)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1950
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1951
    @needs_read_lock
1952
    def has_signature_for_revision_id(self, revision_id):
1953
        """Query for a revision signature for revision_id in the repository."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1954
        if not self.has_revision(revision_id):
1955
            raise errors.NoSuchRevision(self, revision_id)
1956
        sig_present = (1 == len(
1957
            self.signatures.get_parent_map([(revision_id,)])))
1958
        return sig_present
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1959
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1960
    @needs_read_lock
1961
    def get_signature_text(self, revision_id):
1962
        """Return the text for a signature."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1963
        stream = self.signatures.get_record_stream([(revision_id,)],
1964
            'unordered', True)
1965
        record = stream.next()
1966
        if record.storage_kind == 'absent':
1967
            raise errors.NoSuchRevision(self, revision_id)
1968
        return record.get_bytes_as('fulltext')
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
1969
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1970
    @needs_read_lock
2745.6.36 by Andrew Bennetts
Deprecate revision_ids arg to Repository.check and other tweaks.
1971
    def check(self, revision_ids=None):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1972
        """Check consistency of all history of given revision_ids.
1973
1974
        Different repository implementations should override _check().
1975
1976
        :param revision_ids: A non-empty list of revision_ids whose ancestry
1977
             will be checked.  Typically the last revision_id of a branch.
1978
        """
1979
        return self._check(revision_ids)
1980
1981
    def _check(self, revision_ids):
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1982
        result = check.Check(self)
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1983
        result.check()
1984
        return result
1985
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
1986
    def _warn_if_deprecated(self):
1904.2.5 by Martin Pool
Fix format warning inside test suite and add test
1987
        global _deprecation_warning_done
1988
        if _deprecation_warning_done:
1989
            return
1990
        _deprecation_warning_done = True
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
1991
        warning("Format %s for %s is deprecated - please use 'bzr upgrade' to get better performance"
1992
                % (self._format, self.bzrdir.transport.base))
1993
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
1994
    def supports_rich_root(self):
1995
        return self._format.rich_root_data
1996
2150.2.2 by Robert Collins
Change the commit builder selected-revision-id test to use a unicode revision id where possible, leading to stricter testing of the hypothetical unicode revision id support in bzr.
1997
    def _check_ascii_revisionid(self, revision_id, method):
1998
        """Private helper for ascii-only repositories."""
1999
        # weave repositories refuse to store revisionids that are non-ascii.
2000
        if revision_id is not None:
2001
            # weaves require ascii revision ids.
2002
            if isinstance(revision_id, unicode):
2003
                try:
2004
                    revision_id.encode('ascii')
2005
                except UnicodeEncodeError:
2006
                    raise errors.NonAsciiRevisionId(method, self)
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
2007
            else:
2008
                try:
2009
                    revision_id.decode('ascii')
2010
                except UnicodeDecodeError:
2011
                    raise errors.NonAsciiRevisionId(method, self)
2819.2.4 by Andrew Bennetts
Add a 'revision_graph_can_have_wrong_parents' method to repository.
2012
    
2013
    def revision_graph_can_have_wrong_parents(self):
2014
        """Is it possible for this repository to have a revision graph with
2015
        incorrect parents?
2150.2.2 by Robert Collins
Change the commit builder selected-revision-id test to use a unicode revision id where possible, leading to stricter testing of the hypothetical unicode revision id support in bzr.
2016
2819.2.4 by Andrew Bennetts
Add a 'revision_graph_can_have_wrong_parents' method to repository.
2017
        If True, then this repository must also implement
2018
        _find_inconsistent_revision_parents so that check and reconcile can
2019
        check for inconsistencies before proceeding with other checks that may
2020
        depend on the revision index being consistent.
2021
        """
2022
        raise NotImplementedError(self.revision_graph_can_have_wrong_parents)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2023
2024
2241.1.18 by mbp at sourcefrog
Restore use of deprecating delegator for old formats in bzrlib.repository.
2025
# remove these delegates a while after bzr 0.15
2026
def __make_delegated(name, from_module):
2027
    def _deprecated_repository_forwarder():
2028
        symbol_versioning.warn('%s moved to %s in bzr 0.15'
2029
            % (name, from_module),
2241.1.20 by mbp at sourcefrog
update tests for new locations of weave repos
2030
            DeprecationWarning,
2031
            stacklevel=2)
2241.1.18 by mbp at sourcefrog
Restore use of deprecating delegator for old formats in bzrlib.repository.
2032
        m = __import__(from_module, globals(), locals(), [name])
2033
        try:
2034
            return getattr(m, name)
2035
        except AttributeError:
2036
            raise AttributeError('module %s has no name %s'
2037
                    % (m, name))
2038
    globals()[name] = _deprecated_repository_forwarder
2039
2040
for _name in [
2041
        'AllInOneRepository',
2042
        'WeaveMetaDirRepository',
2043
        'PreSplitOutRepositoryFormat',
2044
        'RepositoryFormat4',
2045
        'RepositoryFormat5',
2046
        'RepositoryFormat6',
2047
        'RepositoryFormat7',
2048
        ]:
2049
    __make_delegated(_name, 'bzrlib.repofmt.weaverepo')
2050
2051
for _name in [
2052
        'KnitRepository',
2053
        'RepositoryFormatKnit',
2054
        'RepositoryFormatKnit1',
2055
        ]:
2056
    __make_delegated(_name, 'bzrlib.repofmt.knitrepo')
2057
2058
2996.2.2 by Aaron Bentley
Create install_revisions function
2059
def install_revision(repository, rev, revision_tree):
2060
    """Install all revision data into a repository."""
2061
    install_revisions(repository, [(rev, revision_tree, None)])
2062
2063
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2064
def install_revisions(repository, iterable, num_revisions=None, pb=None):
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
2065
    """Install all revision data into a repository.
2066
2067
    Accepts an iterable of revision, tree, signature tuples.  The signature
2068
    may be None.
2069
    """
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2070
    repository.start_write_group()
2071
    try:
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2072
        for n, (revision, revision_tree, signature) in enumerate(iterable):
2996.2.2 by Aaron Bentley
Create install_revisions function
2073
            _install_revision(repository, revision, revision_tree, signature)
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2074
            if pb is not None:
2075
                pb.update('Transferring revisions', n + 1, num_revisions)
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2076
    except:
2077
        repository.abort_write_group()
2592.3.101 by Robert Collins
Correctly propogate exceptions from repository.install_revisions.
2078
        raise
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2079
    else:
2080
        repository.commit_write_group()
2081
2082
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
2083
def _install_revision(repository, rev, revision_tree, signature):
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2084
    """Install all revision data into a repository."""
1185.82.84 by Aaron Bentley
Moved stuff around
2085
    present_parents = []
2086
    parent_trees = {}
2087
    for p_id in rev.parent_ids:
2088
        if repository.has_revision(p_id):
2089
            present_parents.append(p_id)
2090
            parent_trees[p_id] = repository.revision_tree(p_id)
2091
        else:
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
2092
            parent_trees[p_id] = repository.revision_tree(
2093
                                     _mod_revision.NULL_REVISION)
1185.82.84 by Aaron Bentley
Moved stuff around
2094
2095
    inv = revision_tree.inventory
1910.2.51 by Aaron Bentley
Bundles now corrupt repositories
2096
    entries = inv.iter_entries()
2617.6.6 by Robert Collins
Some review feedback.
2097
    # backwards compatibility hack: skip the root id.
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
2098
    if not repository.supports_rich_root():
1910.2.60 by Aaron Bentley
Ensure that new-model revisions aren't installed into old-model repos
2099
        path, root = entries.next()
2100
        if root.revision != rev.revision_id:
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
2101
            raise errors.IncompatibleRevision(repr(repository))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2102
    text_keys = {}
2103
    for path, ie in entries:
2104
        text_keys[(ie.file_id, ie.revision)] = ie
2105
    text_parent_map = repository.texts.get_parent_map(text_keys)
2106
    missing_texts = set(text_keys) - set(text_parent_map)
1185.82.84 by Aaron Bentley
Moved stuff around
2107
    # Add the texts that are not already present
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2108
    for text_key in missing_texts:
2109
        ie = text_keys[text_key]
2110
        text_parents = []
2111
        # FIXME: TODO: The following loop overlaps/duplicates that done by
2112
        # commit to determine parents. There is a latent/real bug here where
2113
        # the parents inserted are not those commit would do - in particular
2114
        # they are not filtered by heads(). RBC, AB
2115
        for revision, tree in parent_trees.iteritems():
2116
            if ie.file_id not in tree:
2117
                continue
2118
            parent_id = tree.inventory[ie.file_id].revision
2119
            if parent_id in text_parents:
2120
                continue
2121
            text_parents.append((ie.file_id, parent_id))
2122
        lines = revision_tree.get_file(ie.file_id).readlines()
2123
        repository.texts.add_lines(text_key, text_parents, lines)
1185.82.84 by Aaron Bentley
Moved stuff around
2124
    try:
2125
        # install the inventory
2126
        repository.add_inventory(rev.revision_id, inv, present_parents)
2127
    except errors.RevisionAlreadyPresent:
2128
        pass
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
2129
    if signature is not None:
2996.2.8 by Aaron Bentley
Fix add_signature discrepancies
2130
        repository.add_signature_text(rev.revision_id, signature)
1185.82.84 by Aaron Bentley
Moved stuff around
2131
    repository.add_revision(rev.revision_id, rev, inv)
2132
2133
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2134
class MetaDirRepository(Repository):
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
2135
    """Repositories in the new meta-dir layout.
2136
    
2137
    :ivar _transport: Transport for access to repository control files,
2138
        typically pointing to .bzr/repository.
2139
    """
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2140
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2141
    def __init__(self, _format, a_bzrdir, control_files):
2142
        super(MetaDirRepository, self).__init__(_format, a_bzrdir, control_files)
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2143
        self._transport = control_files._transport
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2144
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2145
    def is_shared(self):
2146
        """Return True if this repository is flagged as a shared repository."""
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2147
        return self._transport.has('shared-storage')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2148
2149
    @needs_write_lock
2150
    def set_make_working_trees(self, new_value):
2151
        """Set the policy flag for making working trees when creating branches.
2152
2153
        This only applies to branches that use this repository.
2154
2155
        The default is 'True'.
2156
        :param new_value: True to restore the default, False to disable making
2157
                          working trees.
2158
        """
2159
        if new_value:
2160
            try:
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2161
                self._transport.delete('no-working-trees')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2162
            except errors.NoSuchFile:
2163
                pass
2164
        else:
3407.2.5 by Martin Pool
Deprecate LockableFiles.put_utf8
2165
            self._transport.put_bytes('no-working-trees', '',
3468.1.1 by Martin Pool
Update more users of default file modes from control_files to bzrdir
2166
                mode=self.bzrdir._get_file_mode())
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2167
    
2168
    def make_working_trees(self):
2169
        """Returns the policy for making working trees on new branches."""
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2170
        return not self._transport.has('no-working-trees')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2171
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2172
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
2173
class MetaDirVersionedFileRepository(MetaDirRepository):
2174
    """Repositories in a meta-dir, that work via versioned file objects."""
2175
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2176
    def __init__(self, _format, a_bzrdir, control_files):
3316.2.5 by Robert Collins
Review feedback.
2177
        super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2178
            control_files)
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
2179
2180
2241.1.2 by Martin Pool
change to using external Repository format registry
2181
class RepositoryFormatRegistry(registry.Registry):
2889.1.1 by Robert Collins
* The class ``bzrlib.repofmt.knitrepo.KnitRepository3`` has been folded into
2182
    """Registry of RepositoryFormats."""
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2183
2184
    def get(self, format_string):
2185
        r = registry.Registry.get(self, format_string)
2186
        if callable(r):
2187
            r = r()
2188
        return r
2241.1.2 by Martin Pool
change to using external Repository format registry
2189
    
2190
2191
format_registry = RepositoryFormatRegistry()
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2192
"""Registry of formats, indexed by their identifying format string.
2193
2194
This can contain either format instances themselves, or classes/factories that
2195
can be called to obtain one.
2196
"""
2241.1.2 by Martin Pool
change to using external Repository format registry
2197
2220.2.3 by Martin Pool
Add tag: revision namespace.
2198
2199
#####################################################################
2200
# Repository Formats
1910.2.46 by Aaron Bentley
Whitespace fix
2201
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2202
class RepositoryFormat(object):
2203
    """A repository format.
2204
2205
    Formats provide three things:
2206
     * An initialization routine to construct repository data on disk.
2207
     * a format string which is used when the BzrDir supports versioned
2208
       children.
2209
     * an open routine which returns a Repository instance.
2210
2889.1.2 by Robert Collins
Review feedback.
2211
    There is one and only one Format subclass for each on-disk format. But
2212
    there can be one Repository subclass that is used for several different
2213
    formats. The _format attribute on a Repository instance can be used to
2214
    determine the disk format.
2889.1.1 by Robert Collins
* The class ``bzrlib.repofmt.knitrepo.KnitRepository3`` has been folded into
2215
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2216
    Formats are placed in an dict by their format string for reference 
2217
    during opening. These should be subclasses of RepositoryFormat
2218
    for consistency.
2219
2220
    Once a format is deprecated, just deprecate the initialize and open
2221
    methods on the format class. Do not deprecate the object, as the 
2222
    object will be created every system load.
2223
2224
    Common instance attributes:
2225
    _matchingbzrdir - the bzrdir format that the repository format was
2226
    originally written to work with. This can be used if manually
2227
    constructing a bzrdir and repository, or more commonly for test suite
3128.1.3 by Vincent Ladeuil
Since we are there s/parameteris.*/parameteriz&/.
2228
    parameterization.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2229
    """
2230
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2231
    # Set to True or False in derived classes. True indicates that the format
2232
    # supports ghosts gracefully.
2233
    supports_ghosts = None
3221.3.1 by Robert Collins
* Repository formats have a new supported-feature attribute
2234
    # Can this repository be given external locations to lookup additional
2235
    # data. Set to True or False in derived classes.
2236
    supports_external_lookups = None
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2237
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
2238
    def __str__(self):
2239
        return "<%s>" % self.__class__.__name__
2240
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2241
    def __eq__(self, other):
2242
        # format objects are generally stateless
2243
        return isinstance(other, self.__class__)
2244
2100.3.35 by Aaron Bentley
equality operations on bzrdir
2245
    def __ne__(self, other):
2100.3.31 by Aaron Bentley
Merged bzr.dev (17 tests failing)
2246
        return not self == other
2247
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2248
    @classmethod
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2249
    def find_format(klass, a_bzrdir):
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2250
        """Return the format for the repository object in a_bzrdir.
2251
        
2252
        This is used by bzr native formats that have a "format" file in
2253
        the repository.  Other methods may be used by different types of 
2254
        control directory.
2255
        """
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2256
        try:
2257
            transport = a_bzrdir.get_repository_transport(None)
2258
            format_string = transport.get("format").read()
2241.1.2 by Martin Pool
change to using external Repository format registry
2259
            return format_registry.get(format_string)
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2260
        except errors.NoSuchFile:
2261
            raise errors.NoRepositoryPresent(a_bzrdir)
2262
        except KeyError:
3246.3.2 by Daniel Watkins
Modified uses of errors.UnknownFormatError.
2263
            raise errors.UnknownFormatError(format=format_string,
2264
                                            kind='repository')
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2265
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2266
    @classmethod
2241.1.2 by Martin Pool
change to using external Repository format registry
2267
    def register_format(klass, format):
2268
        format_registry.register(format.get_format_string(), format)
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2269
2270
    @classmethod
2271
    def unregister_format(klass, format):
2241.1.2 by Martin Pool
change to using external Repository format registry
2272
        format_registry.remove(format.get_format_string())
1563.2.23 by Robert Collins
Add add_revision and get_revision methods to RevisionStore
2273
    
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2274
    @classmethod
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2275
    def get_default_format(klass):
2276
        """Return the current default format."""
2204.5.3 by Aaron Bentley
zap old repository default handling
2277
        from bzrlib import bzrdir
2278
        return bzrdir.format_registry.make_bzrdir('default').repository_format
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2279
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2280
    def get_format_string(self):
2281
        """Return the ASCII format string that identifies this format.
2282
        
2283
        Note that in pre format ?? repositories the format string is 
2284
        not permitted nor written to disk.
2285
        """
2286
        raise NotImplementedError(self.get_format_string)
2287
1624.3.19 by Olaf Conradi
New call get_format_description to give a user-friendly description of a
2288
    def get_format_description(self):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
2289
        """Return the short description for this format."""
1624.3.19 by Olaf Conradi
New call get_format_description to give a user-friendly description of a
2290
        raise NotImplementedError(self.get_format_description)
2291
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2292
    # TODO: this shouldn't be in the base class, it's specific to things that
2293
    # use weaves or knits -- mbp 20070207
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2294
    def _get_versioned_file_store(self,
2295
                                  name,
2296
                                  transport,
2297
                                  control_files,
2298
                                  prefixed=True,
2241.1.10 by Martin Pool
Remove more references to weaves from the repository.py file
2299
                                  versionedfile_class=None,
1946.2.5 by John Arbash Meinel
Make knit stores delay creation, but not control stores
2300
                                  versionedfile_kwargs={},
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2301
                                  escaped=False):
2241.1.10 by Martin Pool
Remove more references to weaves from the repository.py file
2302
        if versionedfile_class is None:
2303
            versionedfile_class = self._versionedfile_class
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2304
        weave_transport = control_files._transport.clone(name)
2305
        dir_mode = control_files._dir_mode
2306
        file_mode = control_files._file_mode
2307
        return VersionedFileStore(weave_transport, prefixed=prefixed,
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2308
                                  dir_mode=dir_mode,
2309
                                  file_mode=file_mode,
2310
                                  versionedfile_class=versionedfile_class,
1946.2.5 by John Arbash Meinel
Make knit stores delay creation, but not control stores
2311
                                  versionedfile_kwargs=versionedfile_kwargs,
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2312
                                  escaped=escaped)
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2313
1534.6.1 by Robert Collins
allow API creation of shared repositories
2314
    def initialize(self, a_bzrdir, shared=False):
2315
        """Initialize a repository of this format in a_bzrdir.
2316
2317
        :param a_bzrdir: The bzrdir to put the new repository in it.
2318
        :param shared: The repository should be initialized as a sharable one.
1752.2.52 by Andrew Bennetts
Flesh out more Remote* methods needed to open and initialise remote branches/trees/repositories.
2319
        :returns: The new repository object.
2320
        
1534.6.1 by Robert Collins
allow API creation of shared repositories
2321
        This may raise UninitializableFormat if shared repository are not
2322
        compatible the a_bzrdir.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2323
        """
1752.2.52 by Andrew Bennetts
Flesh out more Remote* methods needed to open and initialise remote branches/trees/repositories.
2324
        raise NotImplementedError(self.initialize)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2325
2326
    def is_supported(self):
2327
        """Is this format supported?
2328
2329
        Supported formats must be initializable and openable.
2330
        Unsupported formats may not support initialization or committing or 
2331
        some other features depending on the reason for not being supported.
2332
        """
2333
        return True
2334
1910.2.12 by Aaron Bentley
Implement knit repo format 2
2335
    def check_conversion_target(self, target_format):
2336
        raise NotImplementedError(self.check_conversion_target)
2337
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2338
    def open(self, a_bzrdir, _found=False):
2339
        """Return an instance of this format for the bzrdir a_bzrdir.
2340
        
2341
        _found is a private parameter, do not use it.
2342
        """
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2343
        raise NotImplementedError(self.open)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2344
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2345
2346
class MetaDirRepositoryFormat(RepositoryFormat):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
2347
    """Common base class for the new repositories using the metadir layout."""
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2348
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2349
    rich_root_data = False
2323.5.17 by Martin Pool
Add supports_tree_reference to all repo formats (robert)
2350
    supports_tree_reference = False
3221.3.1 by Robert Collins
* Repository formats have a new supported-feature attribute
2351
    supports_external_lookups = False
3845.1.1 by John Arbash Meinel
Ensure that RepositoryFormat._matchingbzrdir.repository_format matches.
2352
2353
    @property
2354
    def _matchingbzrdir(self):
2355
        matching = bzrdir.BzrDirMetaFormat1()
2356
        matching.repository_format = self
2357
        return matching
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2358
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
2359
    def __init__(self):
2360
        super(MetaDirRepositoryFormat, self).__init__()
2361
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2362
    def _create_control_files(self, a_bzrdir):
2363
        """Create the required files and the initial control_files object."""
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
2364
        # FIXME: RBC 20060125 don't peek under the covers
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2365
        # NB: no need to escape relative paths that are url safe.
2366
        repository_transport = a_bzrdir.get_repository_transport(self)
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
2367
        control_files = lockable_files.LockableFiles(repository_transport,
2368
                                'lock', lockdir.LockDir)
1553.5.61 by Martin Pool
Locks protecting LockableFiles must now be explicitly created before use.
2369
        control_files.create_lock()
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2370
        return control_files
2371
2372
    def _upload_blank_content(self, a_bzrdir, dirs, files, utf8_files, shared):
2373
        """Upload the initial blank content."""
2374
        control_files = self._create_control_files(a_bzrdir)
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2375
        control_files.lock_write()
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2376
        transport = control_files._transport
2377
        if shared == True:
2378
            utf8_files += [('shared-storage', '')]
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2379
        try:
3446.1.1 by Martin Pool
merge further LockableFile deprecations
2380
            transport.mkdir_multi(dirs, mode=a_bzrdir._get_dir_mode())
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2381
            for (filename, content_stream) in files:
2382
                transport.put_file(filename, content_stream,
3446.1.1 by Martin Pool
merge further LockableFile deprecations
2383
                    mode=a_bzrdir._get_file_mode())
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2384
            for (filename, content_bytes) in utf8_files:
2385
                transport.put_bytes_non_atomic(filename, content_bytes,
3446.1.1 by Martin Pool
merge further LockableFile deprecations
2386
                    mode=a_bzrdir._get_file_mode())
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2387
        finally:
2388
            control_files.unlock()
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2389
2390
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2391
# formats which have no format string are not discoverable
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
2392
# and not independently creatable, so are not registered.  They're 
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2393
# all in bzrlib.repofmt.weaverepo now.  When an instance of one of these is
2394
# needed, it's constructed directly by the BzrDir.  Non-native formats where
2395
# the repository is not separately opened are similar.
2396
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
2397
format_registry.register_lazy(
2398
    'Bazaar-NG Repository format 7',
2399
    'bzrlib.repofmt.weaverepo',
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2400
    'RepositoryFormat7'
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
2401
    )
2592.3.22 by Robert Collins
Add new experimental repository formats.
2402
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2403
format_registry.register_lazy(
2404
    'Bazaar-NG Knit Repository Format 1',
2405
    'bzrlib.repofmt.knitrepo',
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2406
    'RepositoryFormatKnit1',
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2407
    )
2408
2241.1.5 by Martin Pool
Move KnitFormat2 into repofmt
2409
format_registry.register_lazy(
2255.2.230 by Robert Collins
Update tree format signatures to mention introducing bzr version.
2410
    'Bazaar Knit Repository Format 3 (bzr 0.15)\n',
2100.3.31 by Aaron Bentley
Merged bzr.dev (17 tests failing)
2411
    'bzrlib.repofmt.knitrepo',
2412
    'RepositoryFormatKnit3',
2413
    )
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2414
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
2415
format_registry.register_lazy(
2416
    'Bazaar Knit Repository Format 4 (bzr 1.0)\n',
2417
    'bzrlib.repofmt.knitrepo',
2418
    'RepositoryFormatKnit4',
2419
    )
2420
2939.2.1 by Ian Clatworthy
use 'knitpack' naming instead of 'experimental' for pack formats
2421
# Pack-based formats. There is one format for pre-subtrees, and one for
2422
# post-subtrees to allow ease of testing.
3152.2.1 by Robert Collins
* A new repository format 'development' has been added. This format will
2423
# NOTE: These are experimental in 0.92. Stable in 1.0 and above
2592.3.22 by Robert Collins
Add new experimental repository formats.
2424
format_registry.register_lazy(
2939.2.6 by Ian Clatworthy
more review feedback from lifeless and poolie
2425
    'Bazaar pack repository format 1 (needs bzr 0.92)\n',
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2426
    'bzrlib.repofmt.pack_repo',
2592.3.224 by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository
2427
    'RepositoryFormatKnitPack1',
2592.3.22 by Robert Collins
Add new experimental repository formats.
2428
    )
2429
format_registry.register_lazy(
2939.2.6 by Ian Clatworthy
more review feedback from lifeless and poolie
2430
    'Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n',
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2431
    'bzrlib.repofmt.pack_repo',
2592.3.224 by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository
2432
    'RepositoryFormatKnitPack3',
2592.3.22 by Robert Collins
Add new experimental repository formats.
2433
    )
2996.2.11 by Aaron Bentley
Implement rich-root-pack format ( #164639)
2434
format_registry.register_lazy(
2435
    'Bazaar pack repository format 1 with rich root (needs bzr 1.0)\n',
2436
    'bzrlib.repofmt.pack_repo',
2437
    'RepositoryFormatKnitPack4',
2438
    )
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2439
format_registry.register_lazy(
2440
    'Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n',
2441
    'bzrlib.repofmt.pack_repo',
2442
    'RepositoryFormatKnitPack5',
2443
    )
2444
format_registry.register_lazy(
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
2445
    'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n',
2446
    'bzrlib.repofmt.pack_repo',
2447
    'RepositoryFormatKnitPack5RichRoot',
2448
    )
2449
format_registry.register_lazy(
3549.1.6 by Martin Pool
Change stacked-subtree to stacked-rich-root
2450
    'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n',
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2451
    'bzrlib.repofmt.pack_repo',
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
2452
    'RepositoryFormatKnitPack5RichRootBroken',
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2453
    )
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
2454
format_registry.register_lazy(
2455
    'Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n',
2456
    'bzrlib.repofmt.pack_repo',
2457
    'RepositoryFormatKnitPack6',
2458
    )
2459
format_registry.register_lazy(
2460
    'Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n',
2461
    'bzrlib.repofmt.pack_repo',
2462
    'RepositoryFormatKnitPack6RichRoot',
2463
    )
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2464
3152.2.1 by Robert Collins
* A new repository format 'development' has been added. This format will
2465
# Development formats. 
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
2466
# 1.7->1.8 go below here
2467
format_registry.register_lazy(
2468
    "Bazaar development format 2 (needs bzr.dev from before 1.8)\n",
2469
    'bzrlib.repofmt.pack_repo',
2470
    'RepositoryFormatPackDevelopment2',
2471
    )
2472
format_registry.register_lazy(
2473
    ("Bazaar development format 2 with subtree support "
2474
        "(needs bzr.dev from before 1.8)\n"),
2475
    'bzrlib.repofmt.pack_repo',
2476
    'RepositoryFormatPackDevelopment2Subtree',
2477
    )
2592.3.22 by Robert Collins
Add new experimental repository formats.
2478
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2479
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
2480
class InterRepository(InterObject):
1534.1.27 by Robert Collins
Start InterRepository with InterRepository.get.
2481
    """This class represents operations taking place between two repositories.
2482
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2483
    Its instances have methods like copy_content and fetch, and contain
1534.1.27 by Robert Collins
Start InterRepository with InterRepository.get.
2484
    references to the source and target repositories these operations can be 
2485
    carried out on.
2486
2487
    Often we will provide convenience methods on 'repository' which carry out
2488
    operations with another repository - they will always forward to
2489
    InterRepository.get(other).method_name(parameters).
2490
    """
2491
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2492
    _walk_to_common_revisions_batch_size = 1
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2493
    _optimisers = []
1534.1.28 by Robert Collins
Allow for optimised InterRepository selection.
2494
    """The available optimised InterRepository types."""
2495
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
2496
    def __init__(self, source, target):
2497
        InterObject.__init__(self, source, target)
3709.5.5 by Andrew Bennetts
Add comment.
2498
        # These two attributes may be overridden by e.g. InterOtherToRemote to
2499
        # provide a faster implementation.
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
2500
        self.target_get_graph = self.target.get_graph
2501
        self.target_get_parent_map = self.target.get_parent_map
2502
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2503
    def copy_content(self, revision_id=None):
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2504
        raise NotImplementedError(self.copy_content)
2505
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2506
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
2507
        """Fetch the content required to construct revision_id.
2508
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
2509
        The content is copied from self.source to self.target.
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
2510
2511
        :param revision_id: if None all content is copied, if NULL_REVISION no
2512
                            content is copied.
2513
        :param pb: optional progress bar to use for progress reports. If not
2514
                   provided a default one will be created.
2515
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
2516
        :returns: (copied_revision_count, failures).
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
2517
        """
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
2518
        # Normally we should find a specific InterRepository subclass to do
2519
        # the fetch; if nothing else then at least InterSameDataRepository.
2520
        # If none of them is suitable it looks like fetching is not possible;
2521
        # we try to give a good message why.  _assert_same_model will probably
2522
        # give a helpful message; otherwise a generic one.
2523
        self._assert_same_model(self.source, self.target)
2524
        raise errors.IncompatibleRepositories(self.source, self.target,
2525
            "no suitableInterRepository found")
3172.4.4 by Robert Collins
Review feedback.
2526
2527
    def _walk_to_common_revisions(self, revision_ids):
2528
        """Walk out from revision_ids in source to revisions target has.
2529
2530
        :param revision_ids: The start point for the search.
2531
        :return: A set of revision ids.
2532
        """
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
2533
        target_graph = self.target_get_graph()
1551.19.41 by Aaron Bentley
Accelerate no-op pull
2534
        revision_ids = frozenset(revision_ids)
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2535
        # Fast path for the case where all the revisions are already in the
2536
        # target repo.
2537
        # (Although this does incur an extra round trip for the
2538
        # fairly common case where the target doesn't already have the revision
2539
        # we're pushing.)
1551.19.41 by Aaron Bentley
Accelerate no-op pull
2540
        if set(target_graph.get_parent_map(revision_ids)) == revision_ids:
2541
            return graph.SearchResult(revision_ids, set(), 0, set())
3172.4.4 by Robert Collins
Review feedback.
2542
        missing_revs = set()
1551.19.41 by Aaron Bentley
Accelerate no-op pull
2543
        source_graph = self.source.get_graph()
3172.4.4 by Robert Collins
Review feedback.
2544
        # ensure we don't pay silly lookup costs.
1551.19.41 by Aaron Bentley
Accelerate no-op pull
2545
        searcher = source_graph._make_breadth_first_searcher(revision_ids)
3172.4.4 by Robert Collins
Review feedback.
2546
        null_set = frozenset([_mod_revision.NULL_REVISION])
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2547
        searcher_exhausted = False
3172.4.4 by Robert Collins
Review feedback.
2548
        while True:
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2549
            next_revs = set()
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2550
            ghosts = set()
2551
            # Iterate the searcher until we have enough next_revs
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2552
            while len(next_revs) < self._walk_to_common_revisions_batch_size:
2553
                try:
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2554
                    next_revs_part, ghosts_part = searcher.next_with_ghosts()
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2555
                    next_revs.update(next_revs_part)
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2556
                    ghosts.update(ghosts_part)
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2557
                except StopIteration:
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2558
                    searcher_exhausted = True
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
2559
                    break
3731.4.3 by Andrew Bennetts
Rework ghost checking in _walk_to_common_revisions.
2560
            # If there are ghosts in the source graph, and the caller asked for
2561
            # them, make sure that they are present in the target.
3731.4.5 by Andrew Bennetts
Clarify the code slightly.
2562
            # We don't care about other ghosts as we can't fetch them and
2563
            # haven't been asked to.
2564
            ghosts_to_check = set(revision_ids.intersection(ghosts))
2565
            revs_to_get = set(next_revs).union(ghosts_to_check)
2566
            if revs_to_get:
2567
                have_revs = set(target_graph.get_parent_map(revs_to_get))
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2568
                # we always have NULL_REVISION present.
3731.4.5 by Andrew Bennetts
Clarify the code slightly.
2569
                have_revs = have_revs.union(null_set)
2570
                # Check if the target is missing any ghosts we need.
3731.4.3 by Andrew Bennetts
Rework ghost checking in _walk_to_common_revisions.
2571
                ghosts_to_check.difference_update(have_revs)
2572
                if ghosts_to_check:
2573
                    # One of the caller's revision_ids is a ghost in both the
2574
                    # source and the target.
2575
                    raise errors.NoSuchRevision(
2576
                        self.source, ghosts_to_check.pop())
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2577
                missing_revs.update(next_revs - have_revs)
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
2578
                # Because we may have walked past the original stop point, make
2579
                # sure everything is stopped
2580
                stop_revs = searcher.find_seen_ancestors(have_revs)
2581
                searcher.stop_searching_any(stop_revs)
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
2582
            if searcher_exhausted:
3172.4.4 by Robert Collins
Review feedback.
2583
                break
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2584
        return searcher.get_result()
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
2585
3341.2.2 by Alexander Belchenko
Tree.print_file and Repository.print_file are deprecated.
2586
    @deprecated_method(one_two)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2587
    @needs_read_lock
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
2588
    def missing_revision_ids(self, revision_id=None, find_ghosts=True):
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2589
        """Return the revision ids that source has that target does not.
2590
        
2591
        These are returned in topological order.
2592
2593
        :param revision_id: only return revision ids included by this
2594
                            revision_id.
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
2595
        :param find_ghosts: If True find missing revisions in deep history
2596
            rather than just finding the surface difference.
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2597
        """
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2598
        return list(self.search_missing_revision_ids(
2599
            revision_id, find_ghosts).get_keys())
2600
2601
    @needs_read_lock
2602
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
2603
        """Return the revision ids that source has that target does not.
2604
        
2605
        :param revision_id: only return revision ids included by this
2606
                            revision_id.
2607
        :param find_ghosts: If True find missing revisions in deep history
2608
            rather than just finding the surface difference.
2609
        :return: A bzrlib.graph.SearchResult.
2610
        """
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
2611
        # stop searching at found target revisions.
2612
        if not find_ghosts and revision_id is not None:
3172.4.4 by Robert Collins
Review feedback.
2613
            return self._walk_to_common_revisions([revision_id])
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2614
        # generic, possibly worst case, slow code path.
2615
        target_ids = set(self.target.all_revision_ids())
2616
        if revision_id is not None:
2617
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
2618
            if source_ids[0] is not None:
2619
                raise AssertionError()
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2620
            source_ids.pop(0)
2621
        else:
2622
            source_ids = self.source.all_revision_ids()
2623
        result_set = set(source_ids).difference(target_ids)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2624
        return self.source.revision_ids_to_search_result(result_set)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2625
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2626
    @staticmethod
2627
    def _same_model(source, target):
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
2628
        """True if source and target have the same data representation.
2629
        
2630
        Note: this is always called on the base class; overriding it in a
2631
        subclass will have no effect.
2632
        """
2633
        try:
2634
            InterRepository._assert_same_model(source, target)
2635
            return True
2636
        except errors.IncompatibleRepositories, e:
2637
            return False
2638
2639
    @staticmethod
2640
    def _assert_same_model(source, target):
2641
        """Raise an exception if two repositories do not use the same model.
2642
        """
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2643
        if source.supports_rich_root() != target.supports_rich_root():
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
2644
            raise errors.IncompatibleRepositories(source, target,
2645
                "different rich-root support")
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2646
        if source._serializer != target._serializer:
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
2647
            raise errors.IncompatibleRepositories(source, target,
2648
                "different serializers")
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2649
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2650
2651
class InterSameDataRepository(InterRepository):
2652
    """Code for converting between repositories that represent the same data.
2653
    
2654
    Data format and model must match for this to work.
2655
    """
2656
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2657
    @classmethod
2241.1.7 by Martin Pool
rename method
2658
    def _get_repo_format_to_test(self):
2814.1.1 by Robert Collins
* Pushing, pulling and branching branches with subtree references was not
2659
        """Repository format for testing with.
2660
        
2661
        InterSameData can pull from subtree to subtree and from non-subtree to
2662
        non-subtree, so we test this with the richest repository format.
2663
        """
2664
        from bzrlib.repofmt import knitrepo
2665
        return knitrepo.RepositoryFormatKnit3()
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2666
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2667
    @staticmethod
2668
    def is_compatible(source, target):
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2669
        return InterRepository._same_model(source, target)
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2670
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2671
    @needs_write_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2672
    def copy_content(self, revision_id=None):
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2673
        """Make a complete copy of the content in self into destination.
2440.1.1 by Martin Pool
Add new Repository.sprout,
2674
2675
        This copies both the repository's revision data, and configuration information
2676
        such as the make_working_trees setting.
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2677
        
2678
        This is a destructive operation! Do not use it on existing 
2679
        repositories.
2680
2681
        :param revision_id: Only copy the content needed to construct
2682
                            revision_id and its parents.
2683
        """
2684
        try:
2685
            self.target.set_make_working_trees(self.source.make_working_trees())
2686
        except NotImplementedError:
2687
            pass
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
2688
        # but don't bother fetching if we have the needed data now.
1996.3.20 by John Arbash Meinel
[merge] bzr.dev 2063
2689
        if (revision_id not in (None, _mod_revision.NULL_REVISION) and 
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2690
            self.target.has_revision(revision_id)):
2691
            return
2692
        self.target.fetch(self.source, revision_id=revision_id)
2693
2694
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2695
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
1910.7.20 by Andrew Bennetts
Merge from bzr.dev
2696
        """See InterRepository.fetch()."""
3565.3.4 by Robert Collins
Defer decision to reconcile to the repository being fetched into.
2697
        from bzrlib.fetch import RepoFetcher
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
2698
        mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
2858.2.1 by Martin Pool
Remove most calls to safe_file_id and safe_revision_id.
2699
               self.source, self.source._format, self.target,
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2700
               self.target._format)
3565.3.4 by Robert Collins
Defer decision to reconcile to the repository being fetched into.
2701
        f = RepoFetcher(to_repository=self.target,
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2702
                               from_repository=self.source,
2703
                               last_revision=revision_id,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
2704
                               pb=pb, find_ghosts=find_ghosts)
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
2705
        return f.count_copied, f.failed_revisions
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
2706
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2707
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2708
class InterWeaveRepo(InterSameDataRepository):
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
2709
    """Optimised code paths between Weave based repositories.
2710
    
2711
    This should be in bzrlib/repofmt/weaverepo.py but we have not yet
2712
    implemented lazy inter-object optimisation.
2713
    """
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2714
2241.1.13 by Martin Pool
Re-register InterWeaveRepo, fix test integration, add test for it
2715
    @classmethod
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2716
    def _get_repo_format_to_test(self):
2717
        from bzrlib.repofmt import weaverepo
2718
        return weaverepo.RepositoryFormat7()
2719
2720
    @staticmethod
2721
    def is_compatible(source, target):
2722
        """Be compatible with known Weave formats.
2723
        
2724
        We don't test for the stores being of specific types because that
2725
        could lead to confusing results, and there is no need to be 
2726
        overly general.
2727
        """
2728
        from bzrlib.repofmt.weaverepo import (
2729
                RepositoryFormat5,
2730
                RepositoryFormat6,
2731
                RepositoryFormat7,
2732
                )
2733
        try:
2734
            return (isinstance(source._format, (RepositoryFormat5,
2735
                                                RepositoryFormat6,
2736
                                                RepositoryFormat7)) and
2737
                    isinstance(target._format, (RepositoryFormat5,
2738
                                                RepositoryFormat6,
2739
                                                RepositoryFormat7)))
2740
        except AttributeError:
2741
            return False
2742
    
2743
    @needs_write_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2744
    def copy_content(self, revision_id=None):
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2745
        """See InterRepository.copy_content()."""
2746
        # weave specific optimised path:
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2747
        try:
2748
            self.target.set_make_working_trees(self.source.make_working_trees())
3349.1.2 by Aaron Bentley
Change ValueError to RepositoryUpgradeRequired
2749
        except (errors.RepositoryUpgradeRequired, NotImplemented):
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2750
            pass
2751
        # FIXME do not peek!
3407.2.14 by Martin Pool
Remove more cases of getting transport via control_files
2752
        if self.source._transport.listable():
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2753
            pb = ui.ui_factory.nested_progress_bar()
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2754
            try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2755
                self.target.texts.insert_record_stream(
2756
                    self.source.texts.get_record_stream(
2757
                        self.source.texts.keys(), 'topological', False))
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2758
                pb.update('copying inventory', 0, 1)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2759
                self.target.inventories.insert_record_stream(
2760
                    self.source.inventories.get_record_stream(
2761
                        self.source.inventories.keys(), 'topological', False))
2762
                self.target.signatures.insert_record_stream(
2763
                    self.source.signatures.get_record_stream(
2764
                        self.source.signatures.keys(),
2765
                        'unordered', True))
2766
                self.target.revisions.insert_record_stream(
2767
                    self.source.revisions.get_record_stream(
2768
                        self.source.revisions.keys(),
2769
                        'topological', True))
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
2770
            finally:
2771
                pb.finished()
2772
        else:
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2773
            self.target.fetch(self.source, revision_id=revision_id)
2774
2775
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2776
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2777
        """See InterRepository.fetch()."""
3565.3.4 by Robert Collins
Defer decision to reconcile to the repository being fetched into.
2778
        from bzrlib.fetch import RepoFetcher
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2779
        mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
2780
               self.source, self.source._format, self.target, self.target._format)
3565.3.4 by Robert Collins
Defer decision to reconcile to the repository being fetched into.
2781
        f = RepoFetcher(to_repository=self.target,
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2782
                               from_repository=self.source,
2783
                               last_revision=revision_id,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
2784
                               pb=pb, find_ghosts=find_ghosts)
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2785
        return f.count_copied, f.failed_revisions
2786
2787
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2788
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2789
        """See InterRepository.missing_revision_ids()."""
2790
        # we want all revisions to satisfy revision_id in source.
2791
        # but we don't want to stat every file here and there.
2792
        # we want then, all revisions other needs to satisfy revision_id 
2793
        # checked, but not those that we have locally.
2794
        # so the first thing is to get a subset of the revisions to 
2795
        # satisfy revision_id in source, and then eliminate those that
2796
        # we do already have. 
2797
        # this is slow on high latency connection to self, but as as this
2798
        # disk format scales terribly for push anyway due to rewriting 
2799
        # inventory.weave, this is considered acceptable.
2800
        # - RBC 20060209
2801
        if revision_id is not None:
2802
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
2803
            if source_ids[0] is not None:
2804
                raise AssertionError()
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2805
            source_ids.pop(0)
2806
        else:
2807
            source_ids = self.source._all_possible_ids()
2808
        source_ids_set = set(source_ids)
2809
        # source_ids is the worst possible case we may need to pull.
2810
        # now we want to filter source_ids against what we actually
2811
        # have in target, but don't try to check for existence where we know
2812
        # we do not have a revision as that would be pointless.
2813
        target_ids = set(self.target._all_possible_ids())
2814
        possibly_present_revisions = target_ids.intersection(source_ids_set)
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2815
        actually_present_revisions = set(
2816
            self.target._eliminate_revisions_not_present(possibly_present_revisions))
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2817
        required_revisions = source_ids_set.difference(actually_present_revisions)
2818
        if revision_id is not None:
2819
            # we used get_ancestry to determine source_ids then we are assured all
2820
            # revisions referenced are present as they are installed in topological order.
2821
            # and the tip revision was validated by get_ancestry.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2822
            result_set = required_revisions
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2823
        else:
2824
            # if we just grabbed the possibly available ids, then 
2825
            # we only have an estimate of whats available and need to validate
2826
            # that against the revision records.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2827
            result_set = set(
2828
                self.source._eliminate_revisions_not_present(required_revisions))
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2829
        return self.source.revision_ids_to_search_result(result_set)
2241.1.12 by Martin Pool
Restore InterWeaveRepo
2830
2831
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
2832
class InterKnitRepo(InterSameDataRepository):
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2833
    """Optimised code paths between Knit based repositories."""
2834
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2835
    @classmethod
2241.1.7 by Martin Pool
rename method
2836
    def _get_repo_format_to_test(self):
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2837
        from bzrlib.repofmt import knitrepo
2838
        return knitrepo.RepositoryFormatKnit1()
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2839
2840
    @staticmethod
2841
    def is_compatible(source, target):
2842
        """Be compatible with known Knit formats.
2843
        
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
2844
        We don't test for the stores being of specific types because that
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2845
        could lead to confusing results, and there is no need to be 
2846
        overly general.
2847
        """
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2848
        from bzrlib.repofmt.knitrepo import RepositoryFormatKnit
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2849
        try:
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2850
            are_knits = (isinstance(source._format, RepositoryFormatKnit) and
2851
                isinstance(target._format, RepositoryFormatKnit))
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2852
        except AttributeError:
2853
            return False
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
2854
        return are_knits and InterRepository._same_model(source, target)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2855
2856
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2857
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2858
        """See InterRepository.fetch()."""
3565.3.3 by Robert Collins
* Fetching data between repositories that have the same model but no
2859
        from bzrlib.fetch import RepoFetcher
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2860
        mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
2861
               self.source, self.source._format, self.target, self.target._format)
3565.3.3 by Robert Collins
* Fetching data between repositories that have the same model but no
2862
        f = RepoFetcher(to_repository=self.target,
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2863
                            from_repository=self.source,
2864
                            last_revision=revision_id,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
2865
                            pb=pb, find_ghosts=find_ghosts)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2866
        return f.count_copied, f.failed_revisions
2867
2868
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2869
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2870
        """See InterRepository.missing_revision_ids()."""
2871
        if revision_id is not None:
2872
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
2873
            if source_ids[0] is not None:
2874
                raise AssertionError()
1668.1.14 by Martin Pool
merge olaf - InvalidRevisionId fixes
2875
            source_ids.pop(0)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2876
        else:
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
2877
            source_ids = self.source.all_revision_ids()
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2878
        source_ids_set = set(source_ids)
2879
        # source_ids is the worst possible case we may need to pull.
2880
        # now we want to filter source_ids against what we actually
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
2881
        # have in target, but don't try to check for existence where we know
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2882
        # we do not have a revision as that would be pointless.
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
2883
        target_ids = set(self.target.all_revision_ids())
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2884
        possibly_present_revisions = target_ids.intersection(source_ids_set)
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2885
        actually_present_revisions = set(
2886
            self.target._eliminate_revisions_not_present(possibly_present_revisions))
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2887
        required_revisions = source_ids_set.difference(actually_present_revisions)
2888
        if revision_id is not None:
2889
            # we used get_ancestry to determine source_ids then we are assured all
2890
            # revisions referenced are present as they are installed in topological order.
2891
            # and the tip revision was validated by get_ancestry.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2892
            result_set = required_revisions
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2893
        else:
2894
            # if we just grabbed the possibly available ids, then 
2895
            # we only have an estimate of whats available and need to validate
2896
            # that against the revision records.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2897
            result_set = set(
2898
                self.source._eliminate_revisions_not_present(required_revisions))
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2899
        return self.source.revision_ids_to_search_result(result_set)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2900
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
2901
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2902
class InterPackRepo(InterSameDataRepository):
2903
    """Optimised code paths between Pack based repositories."""
2904
2905
    @classmethod
2906
    def _get_repo_format_to_test(self):
2907
        from bzrlib.repofmt import pack_repo
2592.3.224 by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository
2908
        return pack_repo.RepositoryFormatKnitPack1()
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2909
2910
    @staticmethod
2911
    def is_compatible(source, target):
2912
        """Be compatible with known Pack formats.
2913
        
2914
        We don't test for the stores being of specific types because that
2915
        could lead to confusing results, and there is no need to be 
2916
        overly general.
2917
        """
2918
        from bzrlib.repofmt.pack_repo import RepositoryFormatPack
2919
        try:
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
2920
            are_packs = (isinstance(source._format, RepositoryFormatPack) and
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2921
                isinstance(target._format, RepositoryFormatPack))
2922
        except AttributeError:
2923
            return False
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
2924
        return are_packs and InterRepository._same_model(source, target)
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2925
2926
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2927
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2928
        """See InterRepository.fetch()."""
3582.2.1 by Jonathan Lange
Fix up problems with fetching revisions. Almost entirely abentley's work.
2929
        if (len(self.source._fallback_repositories) > 0 or
2930
            len(self.target._fallback_repositories) > 0):
3582.1.13 by Martin Pool
Better comment about fetching to or from stacked repositories
2931
            # The pack layer is not aware of fallback repositories, so when
2932
            # fetching from a stacked repository or into a stacked repository
2933
            # we use the generic fetch logic which uses the VersionedFiles
2934
            # attributes on repository.
3565.3.3 by Robert Collins
* Fetching data between repositories that have the same model but no
2935
            from bzrlib.fetch import RepoFetcher
3834.4.4 by Andrew Bennetts
Add NEWS entry, add comment to the hack in InterPackRepo.fetch.
2936
            # Make sure the generic fetcher sets the write cache size on the
2937
            # new pack (just like Packer.pack does) to avoid doing many tiny
2938
            # writes (which can be slow over a network connection).
2939
            # XXX: ideally the transport layer would do this automatically.
3834.4.3 by Andrew Bennetts
Remove XXX comment, use _get_target_pack_collection helper so that InterPackToRemotePack works.
2940
            pack_coll = self._get_target_pack_collection()
3834.4.1 by Andrew Bennetts
Make InterPackRepo set the packer cache size even when there is stacking.
2941
            set_cache_size = (
2942
                lambda: pack_coll._new_pack.set_write_cache_size(1024*1024))
3565.3.3 by Robert Collins
* Fetching data between repositories that have the same model but no
2943
            fetcher = RepoFetcher(self.target, self.source, revision_id,
3834.4.1 by Andrew Bennetts
Make InterPackRepo set the packer cache size even when there is stacking.
2944
                                  pb, find_ghosts, set_cache_size)
1551.19.46 by Aaron Bentley
Fix fetch from stacked respositories (#248506)
2945
            return fetcher.count_copied, fetcher.failed_revisions
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2946
        mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
2947
               self.source, self.source._format, self.target, self.target._format)
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
2948
        self.count_copied = 0
2949
        if revision_id is None:
2950
            # TODO:
2951
            # everything to do - use pack logic
2952
            # to fetch from all packs to one without
2592.3.93 by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch.
2953
            # inventory parsing etc, IFF nothing to be copied is in the target.
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
2954
            # till then:
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
2955
            source_revision_ids = frozenset(self.source.all_revision_ids())
2956
            revision_ids = source_revision_ids - \
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
2957
                frozenset(self.target_get_parent_map(source_revision_ids))
1551.19.36 by Aaron Bentley
Prevent fetch all from causing pack collisions
2958
            revision_keys = [(revid,) for revid in revision_ids]
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
2959
            target_pack_collection = self._get_target_pack_collection()
2960
            index = target_pack_collection.revision_index.combined_index
1551.19.36 by Aaron Bentley
Prevent fetch all from causing pack collisions
2961
            present_revision_ids = set(item[1][0] for item in
2962
                index.iter_entries(revision_keys))
2963
            revision_ids = set(revision_ids) - present_revision_ids
2592.3.93 by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch.
2964
            # implementing the TODO will involve:
2965
            # - detecting when all of a pack is selected
2966
            # - avoiding as much as possible pre-selection, so the
2967
            # more-core routines such as create_pack_from_packs can filter in
2968
            # a just-in-time fashion. (though having a HEADS list on a
2969
            # repository might make this a lot easier, because we could
2970
            # sensibly detect 'new revisions' without doing a full index scan.
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
2971
        elif _mod_revision.is_null(revision_id):
2972
            # nothing to do:
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
2973
            return (0, [])
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
2974
        else:
2975
            try:
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
2976
                revision_ids = self.search_missing_revision_ids(revision_id,
2977
                    find_ghosts=find_ghosts).get_keys()
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
2978
            except errors.NoSuchRevision:
2979
                raise errors.InstallFailed([revision_id])
1551.19.41 by Aaron Bentley
Accelerate no-op pull
2980
            if len(revision_ids) == 0:
2981
                return (0, [])
3452.2.2 by Andrew Bennetts
Experimental PackRepository.{check_references,autopack} RPCs.
2982
        return self._pack(self.source, self.target, revision_ids)
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
2983
2984
    def _pack(self, source, target, revision_ids):
2985
        from bzrlib.repofmt.pack_repo import Packer
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
2986
        target_pack_collection = self._get_target_pack_collection()
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
2987
        packs = source._pack_collection.all_packs()
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
2988
        pack = Packer(target_pack_collection, packs, '.fetch',
2951.1.2 by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing.
2989
            revision_ids).pack()
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
2990
        if pack is not None:
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
2991
            target_pack_collection._save_pack_names()
2992
            copied_revs = pack.get_revision_count()
2592.3.108 by Robert Collins
Autopack after pack to pack fetching too.
2993
            # Trigger an autopack. This may duplicate effort as we've just done
2994
            # a pack creation, but for now it is simpler to think about as
2995
            # 'upload data, then repack if needed'.
3801.1.16 by Andrew Bennetts
Fix bug introduced in the de-duplication that stopped the autopack RPC from being used.
2996
            self._autopack()
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
2997
            return (copied_revs, [])
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
2998
        else:
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
2999
            return (0, [])
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3000
3801.1.16 by Andrew Bennetts
Fix bug introduced in the de-duplication that stopped the autopack RPC from being used.
3001
    def _autopack(self):
3002
        self.target._pack_collection.autopack()
3003
        
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
3004
    def _get_target_pack_collection(self):
3005
        return self.target._pack_collection
3452.2.4 by Andrew Bennetts
Fix an XXX and remove some commented cruft.
3006
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3007
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3008
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3009
        """See InterRepository.missing_revision_ids().
3010
        
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3011
        :param find_ghosts: Find ghosts throughout the ancestry of
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3012
            revision_id.
3013
        """
3014
        if not find_ghosts and revision_id is not None:
3172.4.4 by Robert Collins
Review feedback.
3015
            return self._walk_to_common_revisions([revision_id])
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3016
        elif revision_id is not None:
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3017
            # Find ghosts: search for revisions pointing from one repository to
3221.18.1 by Ian Clatworthy
tweaks by ianc during review
3018
            # the other, and vice versa, anywhere in the history of revision_id.
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
3019
            graph = self.target_get_graph(other_repository=self.source)
3221.13.7 by Ian Clatworthy
ensure NoSuchRevision raised when needed in InterPackRepo
3020
            searcher = graph._make_breadth_first_searcher([revision_id])
3021
            found_ids = set()
3022
            while True:
3023
                try:
3024
                    next_revs, ghosts = searcher.next_with_ghosts()
3025
                except StopIteration:
3026
                    break
3027
                if revision_id in ghosts:
3028
                    raise errors.NoSuchRevision(self.source, revision_id)
3029
                found_ids.update(next_revs)
3030
                found_ids.update(ghosts)
3031
            found_ids = frozenset(found_ids)
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3032
            # Double query here: should be able to avoid this by changing the
3033
            # graph api further.
3034
            result_set = found_ids - frozenset(
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
3035
                self.target_get_parent_map(found_ids))
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3036
        else:
2592.3.151 by Robert Collins
Use the revision index, not the inventory index, for missing and fetch operations.
3037
            source_ids = self.source.all_revision_ids()
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3038
            # source_ids is the worst possible case we may need to pull.
3039
            # now we want to filter source_ids against what we actually
3040
            # have in target, but don't try to check for existence where we know
3041
            # we do not have a revision as that would be pointless.
3042
            target_ids = set(self.target.all_revision_ids())
3043
            result_set = set(source_ids).difference(target_ids)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3044
        return self.source.revision_ids_to_search_result(result_set)
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3045
3046
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3047
class InterModel1and2(InterRepository):
3048
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3049
    @classmethod
2241.1.7 by Martin Pool
rename method
3050
    def _get_repo_format_to_test(self):
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3051
        return None
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3052
3053
    @staticmethod
3054
    def is_compatible(source, target):
2305.2.1 by Andrew Bennetts
Use repo.supports_rich_root() everywhere rather than
3055
        if not source.supports_rich_root() and target.supports_rich_root():
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3056
            return True
3057
        else:
3058
            return False
3059
3060
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3061
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3062
        """See InterRepository.fetch()."""
3063
        from bzrlib.fetch import Model1toKnit2Fetcher
3064
        f = Model1toKnit2Fetcher(to_repository=self.target,
3065
                                 from_repository=self.source,
3066
                                 last_revision=revision_id,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3067
                                 pb=pb, find_ghosts=find_ghosts)
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3068
        return f.count_copied, f.failed_revisions
3069
1910.2.26 by Aaron Bentley
Fix up some test cases
3070
    @needs_write_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3071
    def copy_content(self, revision_id=None):
1910.2.26 by Aaron Bentley
Fix up some test cases
3072
        """Make a complete copy of the content in self into destination.
3073
        
3074
        This is a destructive operation! Do not use it on existing 
3075
        repositories.
3076
3077
        :param revision_id: Only copy the content needed to construct
3078
                            revision_id and its parents.
3079
        """
3080
        try:
3081
            self.target.set_make_working_trees(self.source.make_working_trees())
3082
        except NotImplementedError:
3083
            pass
3084
        # but don't bother fetching if we have the needed data now.
1996.3.20 by John Arbash Meinel
[merge] bzr.dev 2063
3085
        if (revision_id not in (None, _mod_revision.NULL_REVISION) and 
1910.2.26 by Aaron Bentley
Fix up some test cases
3086
            self.target.has_revision(revision_id)):
3087
            return
3088
        self.target.fetch(self.source, revision_id=revision_id)
3089
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3090
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3091
class InterKnit1and2(InterKnitRepo):
3092
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3093
    @classmethod
2241.1.7 by Martin Pool
rename method
3094
    def _get_repo_format_to_test(self):
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3095
        return None
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3096
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3097
    @staticmethod
3098
    def is_compatible(source, target):
2255.2.211 by Robert Collins
Remove knit2 repository format- it has never been supported.
3099
        """Be compatible with Knit1 source and Knit3 target"""
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3100
        try:
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
3101
            from bzrlib.repofmt.knitrepo import (
3102
                RepositoryFormatKnit1,
3103
                RepositoryFormatKnit3,
3104
                )
3152.2.1 by Robert Collins
* A new repository format 'development' has been added. This format will
3105
            from bzrlib.repofmt.pack_repo import (
3106
                RepositoryFormatKnitPack1,
3107
                RepositoryFormatKnitPack3,
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
3108
                RepositoryFormatKnitPack4,
3109
                RepositoryFormatKnitPack5,
3110
                RepositoryFormatKnitPack5RichRoot,
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
3111
                RepositoryFormatKnitPack6,
3112
                RepositoryFormatKnitPack6RichRoot,
3735.1.2 by Robert Collins
Remove 1.5 series dev formats and document development2 a little better.
3113
                RepositoryFormatPackDevelopment2,
3114
                RepositoryFormatPackDevelopment2Subtree,
3606.14.1 by John Arbash Meinel
Fix bug #264321, properly fetch between non-rich-root formats and rich-root formats.
3115
                )
3116
            norichroot = (
3117
                RepositoryFormatKnit1,            # no rr, no subtree
3118
                RepositoryFormatKnitPack1,        # no rr, no subtree
3735.1.2 by Robert Collins
Remove 1.5 series dev formats and document development2 a little better.
3119
                RepositoryFormatPackDevelopment2, # no rr, no subtree
3606.14.1 by John Arbash Meinel
Fix bug #264321, properly fetch between non-rich-root formats and rich-root formats.
3120
                RepositoryFormatKnitPack5,        # no rr, no subtree
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
3121
                RepositoryFormatKnitPack6,        # no rr, no subtree
3606.14.1 by John Arbash Meinel
Fix bug #264321, properly fetch between non-rich-root formats and rich-root formats.
3122
                )
3123
            richroot = (
3124
                RepositoryFormatKnit3,            # rr, subtree
3125
                RepositoryFormatKnitPack3,        # rr, subtree
3126
                RepositoryFormatKnitPack4,        # rr, no subtree
3127
                RepositoryFormatKnitPack5RichRoot,# rr, no subtree
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
3128
                RepositoryFormatKnitPack6RichRoot,# rr, no subtree
3735.1.2 by Robert Collins
Remove 1.5 series dev formats and document development2 a little better.
3129
                RepositoryFormatPackDevelopment2Subtree, # rr, subtree
3606.14.1 by John Arbash Meinel
Fix bug #264321, properly fetch between non-rich-root formats and rich-root formats.
3130
                )
3131
            for format in norichroot:
3132
                if format.rich_root_data:
3133
                    raise AssertionError('Format %s is a rich-root format'
3134
                        ' but is included in the non-rich-root list'
3135
                        % (format,))
3136
            for format in richroot:
3137
                if not format.rich_root_data:
3138
                    raise AssertionError('Format %s is not a rich-root format'
3139
                        ' but is included in the rich-root list'
3140
                        % (format,))
3141
            # TODO: One alternative is to just check format.rich_root_data,
3142
            #       instead of keeping membership lists. However, the formats
3143
            #       *also* have to use the same 'Knit' style of storage
3144
            #       (line-deltas, fulltexts, etc.)
3145
            return (isinstance(source._format, norichroot) and
3146
                    isinstance(target._format, richroot))
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3147
        except AttributeError:
3148
            return False
3149
3150
    @needs_write_lock
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3151
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3152
        """See InterRepository.fetch()."""
3153
        from bzrlib.fetch import Knit1to2Fetcher
3154
        mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
3155
               self.source, self.source._format, self.target, 
3156
               self.target._format)
3157
        f = Knit1to2Fetcher(to_repository=self.target,
3158
                            from_repository=self.source,
3159
                            last_revision=revision_id,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3160
                            pb=pb, find_ghosts=find_ghosts)
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3161
        return f.count_copied, f.failed_revisions
3162
3163
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3164
class InterDifferingSerializer(InterKnitRepo):
3165
3166
    @classmethod
3167
    def _get_repo_format_to_test(self):
3168
        return None
3169
3170
    @staticmethod
3171
    def is_compatible(source, target):
3172
        """Be compatible with Knit2 source and Knit3 target"""
3173
        if source.supports_rich_root() != target.supports_rich_root():
3174
            return False
3175
        # Ideally, we'd support fetching if the source had no tree references
3176
        # even if it supported them...
3177
        if (getattr(source, '_format.supports_tree_reference', False) and
3178
            not getattr(target, '_format.supports_tree_reference', False)):
3179
            return False
3180
        return True
3181
3182
    @needs_write_lock
3183
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
3184
        """See InterRepository.fetch()."""
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3185
        revision_ids = self.target.search_missing_revision_ids(self.source,
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3186
            revision_id, find_ghosts=find_ghosts).get_keys()
3187
        revision_ids = tsort.topo_sort(
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3188
            self.source.get_graph().get_parent_map(revision_ids))
2996.2.2 by Aaron Bentley
Create install_revisions function
3189
        def revisions_iterator():
3831.2.1 by Andrew Bennetts
Quick hack to do batching in InterDifferingSerializer. Almost halves the HPSS round-trips fetching pack-0.92-subtree to 1.9-rich-root.
3190
            rev_ids = list(revision_ids)
3191
            for offset in xrange(0, len(rev_ids), 100):
3192
                current_revids = rev_ids[offset:offset+100]
3193
                revisions = self.source.get_revisions(current_revids)
3194
                trees = self.source.revision_trees(current_revids)
3195
                keys = [(r,) for r in current_revids]
3196
                sig_stream = self.source.signatures.get_record_stream(
3197
                    keys, 'unordered', True)
3198
                sigs = {}
3199
                for record in versionedfile.filter_absent(sig_stream):
3200
                    sigs[record.key[0]] = record.get_bytes_as('fulltext')
3201
                for rev, tree in zip(revisions, trees):
3202
                    yield rev, tree, sigs.get(rev.revision_id, None)
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
3203
        if pb is None:
3204
            my_pb = ui.ui_factory.nested_progress_bar()
3205
            pb = my_pb
3206
        else:
3207
            my_pb = None
3208
        try:
3209
            install_revisions(self.target, revisions_iterator(),
3210
                              len(revision_ids), pb)
3211
        finally:
3212
            if my_pb is not None:
3213
                my_pb.finished()
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3214
        return len(revision_ids), 0
3215
3216
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3217
class InterOtherToRemote(InterRepository):
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3218
    """An InterRepository that simply delegates to the 'real' InterRepository
3219
    calculated for (source, target._real_repository).
3220
    """
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3221
3731.4.1 by Andrew Bennetts
Extract InterOtherToRemoteRepo get_parent_map batching from push-improvement-loom.
3222
    _walk_to_common_revisions_batch_size = 50
3223
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3224
    def __init__(self, source, target):
3225
        InterRepository.__init__(self, source, target)
3226
        self._real_inter = None
3227
3228
    @staticmethod
3229
    def is_compatible(source, target):
3230
        if isinstance(target, remote.RemoteRepository):
3231
            return True
3232
        return False
3233
3234
    def _ensure_real_inter(self):
3235
        if self._real_inter is None:
3236
            self.target._ensure_real()
3237
            real_target = self.target._real_repository
3238
            self._real_inter = InterRepository.get(self.source, real_target)
3709.5.1 by Andrew Bennetts
Allow pushing to a pack repo over HPSS use the get_parent_map RPC, and teach the get_parent_map client to cache missing revisions.
3239
            # Make _real_inter use the RemoteRepository for get_parent_map
3240
            self._real_inter.target_get_graph = self.target.get_graph
3241
            self._real_inter.target_get_parent_map = self.target.get_parent_map
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3242
    
3243
    def copy_content(self, revision_id=None):
3244
        self._ensure_real_inter()
3245
        self._real_inter.copy_content(revision_id=revision_id)
3246
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3247
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3248
        self._ensure_real_inter()
3453.5.1 by Andrew Bennetts
Add {bzrdir,repository,branch}_implementations tests for Remote objects using protocol v2 and pre-1.6 RPCs.
3249
        return self._real_inter.fetch(revision_id=revision_id, pb=pb,
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3250
            find_ghosts=find_ghosts)
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3251
3252
    @classmethod
3253
    def _get_repo_format_to_test(self):
3254
        return None
3255
3256
3515.2.1 by Andrew Bennetts
Reinstate InterRemoteToOther. This allows pushes and pulls from a remote repository to use the best InterRepository for the underlying format.
3257
class InterRemoteToOther(InterRepository):
3258
3259
    def __init__(self, source, target):
3260
        InterRepository.__init__(self, source, target)
3261
        self._real_inter = None
3262
3263
    @staticmethod
3264
    def is_compatible(source, target):
3265
        if not isinstance(source, remote.RemoteRepository):
3266
            return False
3267
        # Is source's model compatible with target's model?
3268
        source._ensure_real()
3269
        real_source = source._real_repository
3270
        if isinstance(real_source, remote.RemoteRepository):
3271
            raise NotImplementedError(
3272
                "We don't support remote repos backed by remote repos yet.")
3273
        return InterRepository._same_model(real_source, target)
3274
3275
    def _ensure_real_inter(self):
3276
        if self._real_inter is None:
3277
            self.source._ensure_real()
3278
            real_source = self.source._real_repository
3279
            self._real_inter = InterRepository.get(real_source, self.target)
3280
    
3281
    def fetch(self, revision_id=None, pb=None, find_ghosts=False):
3282
        self._ensure_real_inter()
3283
        return self._real_inter.fetch(revision_id=revision_id, pb=pb,
3284
            find_ghosts=find_ghosts)
3285
3286
    def copy_content(self, revision_id=None):
3287
        self._ensure_real_inter()
3288
        self._real_inter.copy_content(revision_id=revision_id)
3289
3290
    @classmethod
3291
    def _get_repo_format_to_test(self):
3292
        return None
3293
3294
3295
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3296
class InterPackToRemotePack(InterPackRepo):
3297
    """A specialisation of InterPackRepo for a target that is a
3298
    RemoteRepository.
3299
3452.2.8 by Andrew Bennetts
Tidy some cruft.
3300
    This will use the get_parent_map RPC rather than plain readvs, and also
3801.1.6 by Andrew Bennetts
Remove check_references RPC for now.
3301
    uses an RPC for autopacking.
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3302
    """
3303
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3304
    _walk_to_common_revisions_batch_size = 50
3305
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3306
    @staticmethod
3307
    def is_compatible(source, target):
3308
        from bzrlib.repofmt.pack_repo import RepositoryFormatPack
3309
        if isinstance(source._format, RepositoryFormatPack):
3310
            if isinstance(target, remote.RemoteRepository):
3311
                target._ensure_real()
3312
                if isinstance(target._real_repository._format,
3313
                              RepositoryFormatPack):
3314
                    if InterRepository._same_model(source, target):
3315
                        return True
3316
        return False
3452.2.4 by Andrew Bennetts
Fix an XXX and remove some commented cruft.
3317
    
3801.1.16 by Andrew Bennetts
Fix bug introduced in the de-duplication that stopped the autopack RPC from being used.
3318
    def _autopack(self):
3319
        self.target.autopack()
3320
        
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
3321
    def _get_target_pack_collection(self):
3322
        return self.target._real_repository._pack_collection
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3323
3324
    @classmethod
3325
    def _get_repo_format_to_test(self):
3326
        return None
3327
3328
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3329
InterRepository.register_optimiser(InterDifferingSerializer)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3330
InterRepository.register_optimiser(InterSameDataRepository)
2241.1.13 by Martin Pool
Re-register InterWeaveRepo, fix test integration, add test for it
3331
InterRepository.register_optimiser(InterWeaveRepo)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3332
InterRepository.register_optimiser(InterKnitRepo)
1910.2.24 by Aaron Bentley
Got intra-repository fetch working between model1 and 2 for all types
3333
InterRepository.register_optimiser(InterModel1and2)
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3334
InterRepository.register_optimiser(InterKnit1and2)
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3335
InterRepository.register_optimiser(InterPackRepo)
2535.3.12 by Andrew Bennetts
Add a first cut of a get_data_stream method to Repository.
3336
InterRepository.register_optimiser(InterOtherToRemote)
3515.2.1 by Andrew Bennetts
Reinstate InterRemoteToOther. This allows pushes and pulls from a remote repository to use the best InterRepository for the underlying format.
3337
InterRepository.register_optimiser(InterRemoteToOther)
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3338
InterRepository.register_optimiser(InterPackToRemotePack)
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
3339
3340
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3341
class CopyConverter(object):
3342
    """A repository conversion tool which just performs a copy of the content.
3343
    
3344
    This is slow but quite reliable.
3345
    """
3346
3347
    def __init__(self, target_format):
3348
        """Create a CopyConverter.
3349
3350
        :param target_format: The format the resulting repository should be.
3351
        """
3352
        self.target_format = target_format
3353
        
3354
    def convert(self, repo, pb):
3355
        """Perform the conversion of to_convert, giving feedback via pb.
3356
3357
        :param to_convert: The disk object to convert.
3358
        :param pb: a progress bar to use for progress information.
3359
        """
3360
        self.pb = pb
3361
        self.count = 0
1596.2.22 by Robert Collins
Fetch changes to use new pb.
3362
        self.total = 4
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3363
        # this is only useful with metadir layouts - separated repo content.
3364
        # trigger an assertion if not such
3365
        repo._format.get_format_string()
3366
        self.repo_dir = repo.bzrdir
3367
        self.step('Moving repository to repository.backup')
3368
        self.repo_dir.transport.move('repository', 'repository.backup')
3369
        backup_transport =  self.repo_dir.transport.clone('repository.backup')
1910.2.12 by Aaron Bentley
Implement knit repo format 2
3370
        repo._format.check_conversion_target(self.target_format)
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3371
        self.source_repo = repo._format.open(self.repo_dir,
3372
            _found=True,
3373
            _override_transport=backup_transport)
3374
        self.step('Creating new repository')
3375
        converted = self.target_format.initialize(self.repo_dir,
3376
                                                  self.source_repo.is_shared())
3377
        converted.lock_write()
3378
        try:
3379
            self.step('Copying content into repository.')
3380
            self.source_repo.copy_content_into(converted)
3381
        finally:
3382
            converted.unlock()
3383
        self.step('Deleting old repository content.')
3384
        self.repo_dir.transport.delete_tree('repository.backup')
3385
        self.pb.note('repository converted')
3386
3387
    def step(self, message):
3388
        """Update the pb by a step."""
3389
        self.count +=1
3390
        self.pb.update(message, self.count, self.total)
1596.1.1 by Martin Pool
Use simple xml unescaping rather than importing xml.sax
3391
3392
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3393
_unescape_map = {
3394
    'apos':"'",
3395
    'quot':'"',
3396
    'amp':'&',
3397
    'lt':'<',
3398
    'gt':'>'
3399
}
3400
3401
3402
def _unescaper(match, _map=_unescape_map):
2294.1.2 by John Arbash Meinel
Track down and add tests that all tree.commit() can handle
3403
    code = match.group(1)
3404
    try:
3405
        return _map[code]
3406
    except KeyError:
3407
        if not code.startswith('#'):
3408
            raise
2294.1.10 by John Arbash Meinel
Switch all apis over to utf8 file ids. All tests pass
3409
        return unichr(int(code[1:])).encode('utf8')
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3410
3411
3412
_unescape_re = None
3413
3414
1596.1.1 by Martin Pool
Use simple xml unescaping rather than importing xml.sax
3415
def _unescape_xml(data):
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3416
    """Unescape predefined XML entities in a string of data."""
3417
    global _unescape_re
3418
    if _unescape_re is None:
2120.2.1 by John Arbash Meinel
Remove tabs from source files, and add a test to keep it that way.
3419
        _unescape_re = re.compile('\&([^;]*);')
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3420
    return _unescape_re.sub(_unescaper, data)
2745.6.3 by Aaron Bentley
Implement versionedfile checking for bzr check
3421
3422
3036.1.3 by Robert Collins
Privatise VersionedFileChecker.
3423
class _VersionedFileChecker(object):
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3424
2988.1.6 by Robert Collins
Change the contract for VersionedFileChecker to consolidate related parameters rather than splitting them across two api calls. This allows better reuse of a single checker object.
3425
    def __init__(self, repository):
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3426
        self.repository = repository
2988.1.8 by Robert Collins
Change check and reconcile to use the new _generate_text_key_index rather
3427
        self.text_index = self.repository._generate_text_key_index()
2745.6.49 by Andrew Bennetts
Get rid of bzrlib.repository._RevisionParentsProvider.
3428
    
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3429
    def calculate_file_version_parents(self, text_key):
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3430
        """Calculate the correct parents for a file version according to
3431
        the inventories.
3432
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3433
        parent_keys = self.text_index[text_key]
2988.1.8 by Robert Collins
Change check and reconcile to use the new _generate_text_key_index rather
3434
        if parent_keys == [_mod_revision.NULL_REVISION]:
3435
            return ()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3436
        return tuple(parent_keys)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3437
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3438
    def check_file_version_parents(self, texts, progress_bar=None):
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3439
        """Check the parents stored in a versioned file are correct.
3440
3441
        It also detects file versions that are not referenced by their
3442
        corresponding revision's inventory.
3443
2927.2.14 by Andrew Bennetts
Tweaks suggested by review.
3444
        :returns: A tuple of (wrong_parents, dangling_file_versions).
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3445
            wrong_parents is a dict mapping {revision_id: (stored_parents,
3446
            correct_parents)} for each revision_id where the stored parents
2927.2.14 by Andrew Bennetts
Tweaks suggested by review.
3447
            are not correct.  dangling_file_versions is a set of (file_id,
3448
            revision_id) tuples for versions that are present in this versioned
3449
            file, but not used by the corresponding inventory.
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3450
        """
2927.2.3 by Andrew Bennetts
Add fulltexts to avoid bug 155730.
3451
        wrong_parents = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3452
        self.file_ids = set([file_id for file_id, _ in
3453
            self.text_index.iterkeys()])
3454
        # text keys is now grouped by file_id
3455
        n_weaves = len(self.file_ids)
3456
        files_in_revisions = {}
3457
        revisions_of_files = {}
3458
        n_versions = len(self.text_index)
3459
        progress_bar.update('loading text store', 0, n_versions)
3460
        parent_map = self.repository.texts.get_parent_map(self.text_index)
3461
        # On unlistable transports this could well be empty/error...
3462
        text_keys = self.repository.texts.keys()
3463
        unused_keys = frozenset(text_keys) - set(self.text_index)
3464
        for num, key in enumerate(self.text_index.iterkeys()):
3465
            if progress_bar is not None:
3466
                progress_bar.update('checking text graph', num, n_versions)
3467
            correct_parents = self.calculate_file_version_parents(key)
2927.2.6 by Andrew Bennetts
Make some more check tests pass.
3468
            try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3469
                knit_parents = parent_map[key]
3470
            except errors.RevisionNotPresent:
3471
                # Missing text!
3472
                knit_parents = None
3473
            if correct_parents != knit_parents:
3474
                wrong_parents[key] = (knit_parents, correct_parents)
3475
        return wrong_parents, unused_keys
3287.6.8 by Robert Collins
Reduce code duplication as per review.
3476
3477
3478
def _old_get_graph(repository, revision_id):
3479
    """DO NOT USE. That is all. I'm serious."""
3480
    graph = repository.get_graph()
3481
    revision_graph = dict(((key, value) for key, value in
3482
        graph.iter_ancestry([revision_id]) if value is not None))
3483
    return _strip_NULL_ghosts(revision_graph)
3484
3485
3486
def _strip_NULL_ghosts(revision_graph):
3487
    """Also don't use this. more compatibility code for unmigrated clients."""
3488
    # Filter ghosts, and null:
3489
    if _mod_revision.NULL_REVISION in revision_graph:
3490
        del revision_graph[_mod_revision.NULL_REVISION]
3491
    for key, parents in revision_graph.items():
3492
        revision_graph[key] = tuple(parent for parent in parents if parent
3493
            in revision_graph)
3494
    return revision_graph