/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
4110.2.1 by Martin Pool
Remove duplicated code from InterRepository implementations.
1
# Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.65.10 by Robert Collins
Rename Controlfiles to LockableFiles.
16
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
17
from bzrlib.lazy_import import lazy_import
18
lazy_import(globals(), """
4232.2.1 by Vincent Ladeuil
Stop-gap fix for Repository.get_revision_xml.
19
import cStringIO
1740.3.7 by Jelmer Vernooij
Move committer, log, revprops, timestamp and timezone to CommitBuilder.
20
import re
21
import time
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
22
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
23
from bzrlib import (
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
24
    bzrdir,
25
    check,
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
26
    chk_map,
2745.1.1 by Robert Collins
Add a number of -Devil checkpoints.
27
    debug,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
28
    errors,
3882.6.23 by John Arbash Meinel
Change the XMLSerializer.read_inventory_from_string api.
29
    fifo_cache,
2116.4.1 by John Arbash Meinel
Update file and revision id generators.
30
    generate_ids,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
31
    gpg,
32
    graph,
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
33
    inventory,
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
34
    lazy_regex,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
35
    lockable_files,
36
    lockdir,
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
37
    lru_cache,
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
38
    osutils,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
39
    revision as _mod_revision,
40
    symbol_versioning,
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
41
    tsort,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
42
    ui,
3831.2.1 by Andrew Bennetts
Quick hack to do batching in InterDifferingSerializer. Almost halves the HPSS round-trips fetching pack-0.92-subtree to 1.9-rich-root.
43
    versionedfile,
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
44
    )
2520.4.54 by Aaron Bentley
Hang a create_bundle method off repository
45
from bzrlib.bundle import serializer
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
46
from bzrlib.revisiontree import RevisionTree
47
from bzrlib.store.versioned import VersionedFileStore
48
from bzrlib.testament import Testament
49
""")
50
1534.4.28 by Robert Collins
first cut at merge from integration.
51
from bzrlib.decorators import needs_read_lock, needs_write_lock
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
52
from bzrlib.inter import InterObject
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
53
from bzrlib.inventory import (
54
    Inventory,
55
    InventoryDirectory,
56
    ROOT_ID,
57
    entry_factory,
58
    )
4032.3.1 by Robert Collins
Add a BranchFormat.network_name() method as preparation for creating branches via RPC calls.
59
from bzrlib import registry
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
60
from bzrlib.symbol_versioning import (
61
        deprecated_method,
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
62
        )
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
63
from bzrlib.trace import (
64
    log_exception_quietly, note, mutter, mutter_callsite, warning)
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
65
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
66
1904.2.5 by Martin Pool
Fix format warning inside test suite and add test
67
# Old formats display a warning, but only once
68
_deprecation_warning_done = False
69
70
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
71
class CommitBuilder(object):
72
    """Provides an interface to build up a commit.
73
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
74
    This allows describing a tree to be committed without needing to
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
75
    know the internals of the format of the repository.
76
    """
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
77
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
78
    # all clients should supply tree roots.
79
    record_root_entry = True
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
80
    # the default CommitBuilder does not manage trees whose root is versioned.
81
    _versioned_root = False
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
82
2979.2.2 by Robert Collins
Per-file graph heads detection during commit for pack repositories.
83
    def __init__(self, repository, parents, config, timestamp=None,
84
                 timezone=None, committer=None, revprops=None,
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
85
                 revision_id=None):
86
        """Initiate a CommitBuilder.
87
88
        :param repository: Repository to commit to.
89
        :param parents: Revision ids of the parents of the new revision.
90
        :param config: Configuration to use.
91
        :param timestamp: Optional timestamp recorded for commit.
92
        :param timezone: Optional timezone for timestamp.
93
        :param committer: Optional committer to set for commit.
94
        :param revprops: Optional dictionary of revision properties.
95
        :param revision_id: Optional revision id.
96
        """
97
        self._config = config
98
99
        if committer is None:
100
            self._committer = self._config.username()
101
        else:
102
            self._committer = committer
103
104
        self.new_inventory = Inventory(None)
2858.2.1 by Martin Pool
Remove most calls to safe_file_id and safe_revision_id.
105
        self._new_revision_id = revision_id
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
106
        self.parents = parents
107
        self.repository = repository
108
109
        self._revprops = {}
110
        if revprops is not None:
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
111
            self._validate_revprops(revprops)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
112
            self._revprops.update(revprops)
113
114
        if timestamp is None:
115
            timestamp = time.time()
116
        # Restrict resolution to 1ms
117
        self._timestamp = round(timestamp, 3)
118
119
        if timezone is None:
120
            self._timezone = osutils.local_time_offset()
121
        else:
122
            self._timezone = int(timezone)
123
124
        self._generate_revision_if_needed()
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
125
        self.__heads = graph.HeadsCache(repository.get_graph()).heads
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
126
        self._basis_delta = []
127
        # API compatibility, older code that used CommitBuilder did not call
128
        # .record_delete(), which means the delta that is computed would not be
129
        # valid. Callers that will call record_delete() should call
130
        # .will_record_deletes() to indicate that.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
131
        self._recording_deletes = False
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
132
        # memo'd check for no-op commits.
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
133
        self._any_changes = False
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
134
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
135
    def any_changes(self):
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
136
        """Return True if any entries were changed.
137
        
138
        This includes merge-only changes. It is the core for the --unchanged
139
        detection in commit.
140
141
        :return: True if any changes have occured.
142
        """
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
143
        return self._any_changes
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
144
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
145
    def _validate_unicode_text(self, text, context):
146
        """Verify things like commit messages don't have bogus characters."""
147
        if '\r' in text:
148
            raise ValueError('Invalid value for %s: %r' % (context, text))
149
150
    def _validate_revprops(self, revprops):
151
        for key, value in revprops.iteritems():
152
            # We know that the XML serializers do not round trip '\r'
153
            # correctly, so refuse to accept them
3831.1.5 by John Arbash Meinel
It seems we have some direct tests that don't use strings and expect a value error as well.
154
            if not isinstance(value, basestring):
155
                raise ValueError('revision property (%s) is not a valid'
156
                                 ' (unicode) string: %r' % (key, value))
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
157
            self._validate_unicode_text(value,
158
                                        'revision property (%s)' % (key,))
159
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
160
    def commit(self, message):
161
        """Make the actual commit.
162
163
        :return: The revision id of the recorded revision.
164
        """
3831.1.1 by John Arbash Meinel
Before allowing commit to succeed, verify the texts will be 'safe'.
165
        self._validate_unicode_text(message, 'commit message')
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
166
        rev = _mod_revision.Revision(
167
                       timestamp=self._timestamp,
168
                       timezone=self._timezone,
169
                       committer=self._committer,
170
                       message=message,
171
                       inventory_sha1=self.inv_sha1,
172
                       revision_id=self._new_revision_id,
173
                       properties=self._revprops)
174
        rev.parent_ids = self.parents
175
        self.repository.add_revision(self._new_revision_id, rev,
176
            self.new_inventory, self._config)
177
        self.repository.commit_write_group()
178
        return self._new_revision_id
179
180
    def abort(self):
181
        """Abort the commit that is being built.
182
        """
183
        self.repository.abort_write_group()
184
185
    def revision_tree(self):
186
        """Return the tree that was just committed.
187
188
        After calling commit() this can be called to get a RevisionTree
189
        representing the newly committed tree. This is preferred to
190
        calling Repository.revision_tree() because that may require
191
        deserializing the inventory, while we already have a copy in
192
        memory.
193
        """
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
194
        if self.new_inventory is None:
195
            self.new_inventory = self.repository.get_inventory(
196
                self._new_revision_id)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
197
        return RevisionTree(self.repository, self.new_inventory,
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
198
            self._new_revision_id)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
199
200
    def finish_inventory(self):
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
201
        """Tell the builder that the inventory is finished.
3735.2.163 by John Arbash Meinel
Merge bzr.dev 4187, and revert the change to fix refcycle issues.
202
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
203
        :return: The inventory id in the repository, which can be used with
204
            repository.get_inventory.
205
        """
206
        if self.new_inventory is None:
207
            # an inventory delta was accumulated without creating a new
3735.2.12 by Robert Collins
Implement commit-via-deltas for split inventory repositories.
208
            # inventory.
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
209
            basis_id = self.basis_delta_revision
210
            self.inv_sha1 = self.repository.add_inventory_by_delta(
211
                basis_id, self._basis_delta, self._new_revision_id,
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
212
                self.parents)
3735.2.12 by Robert Collins
Implement commit-via-deltas for split inventory repositories.
213
        else:
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
214
            if self.new_inventory.root is None:
215
                raise AssertionError('Root entry should be supplied to'
216
                    ' record_entry_contents, as of bzr 0.10.')
217
                self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
218
            self.new_inventory.revision_id = self._new_revision_id
219
            self.inv_sha1 = self.repository.add_inventory(
220
                self._new_revision_id,
221
                self.new_inventory,
222
                self.parents
223
                )
224
        return self._new_revision_id
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
225
226
    def _gen_revision_id(self):
227
        """Return new revision-id."""
228
        return generate_ids.gen_revision_id(self._config.username(),
229
                                            self._timestamp)
230
231
    def _generate_revision_if_needed(self):
232
        """Create a revision id if None was supplied.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
233
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
234
        If the repository can not support user-specified revision ids
235
        they should override this function and raise CannotSetRevisionId
236
        if _new_revision_id is not None.
237
238
        :raises: CannotSetRevisionId
239
        """
240
        if self._new_revision_id is None:
241
            self._new_revision_id = self._gen_revision_id()
242
            self.random_revid = True
243
        else:
244
            self.random_revid = False
245
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
246
    def _heads(self, file_id, revision_ids):
2979.2.1 by Robert Collins
Make it possible for different commit builders to override heads().
247
        """Calculate the graph heads for revision_ids in the graph of file_id.
248
249
        This can use either a per-file graph or a global revision graph as we
250
        have an identity relationship between the two graphs.
251
        """
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
252
        return self.__heads(revision_ids)
2979.2.1 by Robert Collins
Make it possible for different commit builders to override heads().
253
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
254
    def _check_root(self, ie, parent_invs, tree):
255
        """Helper for record_entry_contents.
256
257
        :param ie: An entry being added.
258
        :param parent_invs: The inventories of the parent revisions of the
259
            commit.
260
        :param tree: The tree that is being committed.
261
        """
2871.1.2 by Robert Collins
* ``CommitBuilder.record_entry_contents`` now requires the root entry of a
262
        # In this revision format, root entries have no knit or weave When
263
        # serializing out to disk and back in root.revision is always
264
        # _new_revision_id
265
        ie.revision = self._new_revision_id
2818.3.1 by Robert Collins
Change CommitBuilder factory delegation to allow simple declaration.
266
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
267
    def _require_root_change(self, tree):
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
268
        """Enforce an appropriate root object change.
269
270
        This is called once when record_iter_changes is called, if and only if
271
        the root was not in the delta calculated by record_iter_changes.
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
272
273
        :param tree: The tree which is being committed.
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
274
        """
275
        # NB: if there are no parents then this method is not called, so no
276
        # need to guard on parents having length.
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
277
        entry = entry_factory['directory'](tree.path2id(''), '',
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
278
            None)
279
        entry.revision = self._new_revision_id
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
280
        self._basis_delta.append(('', '', entry.file_id, entry))
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
281
2871.1.4 by Robert Collins
Merge bzr.dev.
282
    def _get_delta(self, ie, basis_inv, path):
283
        """Get a delta against the basis inventory for ie."""
284
        if ie.file_id not in basis_inv:
285
            # add
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
286
            result = (None, path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
287
            self._basis_delta.append(result)
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
288
            return result
2871.1.4 by Robert Collins
Merge bzr.dev.
289
        elif ie != basis_inv[ie.file_id]:
290
            # common but altered
291
            # TODO: avoid tis id2path call.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
292
            result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
293
            self._basis_delta.append(result)
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
294
            return result
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
295
        else:
2871.1.4 by Robert Collins
Merge bzr.dev.
296
            # common, unaltered
297
            return None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
298
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
299
    def get_basis_delta(self):
300
        """Return the complete inventory delta versus the basis inventory.
301
302
        This has been built up with the calls to record_delete and
303
        record_entry_contents. The client must have already called
304
        will_record_deletes() to indicate that they will be generating a
305
        complete delta.
306
307
        :return: An inventory delta, suitable for use with apply_delta, or
308
            Repository.add_inventory_by_delta, etc.
309
        """
310
        if not self._recording_deletes:
311
            raise AssertionError("recording deletes not activated.")
312
        return self._basis_delta
313
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
314
    def record_delete(self, path, file_id):
315
        """Record that a delete occured against a basis tree.
316
317
        This is an optional API - when used it adds items to the basis_delta
318
        being accumulated by the commit builder. It cannot be called unless the
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
319
        method will_record_deletes() has been called to inform the builder that
320
        a delta is being supplied.
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
321
322
        :param path: The path of the thing deleted.
323
        :param file_id: The file id that was deleted.
324
        """
325
        if not self._recording_deletes:
326
            raise AssertionError("recording deletes not activated.")
3879.2.5 by John Arbash Meinel
Change record_delete() to return the delta.
327
        delta = (path, None, file_id, None)
328
        self._basis_delta.append(delta)
4183.5.5 by Robert Collins
Enable record_iter_changes for cases where it can work.
329
        self._any_changes = True
3879.2.5 by John Arbash Meinel
Change record_delete() to return the delta.
330
        return delta
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
331
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
332
    def will_record_deletes(self):
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
333
        """Tell the commit builder that deletes are being notified.
334
335
        This enables the accumulation of an inventory delta; for the resulting
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
336
        commit to be valid, deletes against the basis MUST be recorded via
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
337
        builder.record_delete().
338
        """
339
        self._recording_deletes = True
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
340
        try:
341
            basis_id = self.parents[0]
342
        except IndexError:
343
            basis_id = _mod_revision.NULL_REVISION
344
        self.basis_delta_revision = basis_id
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
345
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
346
    def record_entry_contents(self, ie, parent_invs, path, tree,
347
        content_summary):
348
        """Record the content of ie from tree into the commit if needed.
349
350
        Side effect: sets ie.revision when unchanged
351
352
        :param ie: An inventory entry present in the commit.
353
        :param parent_invs: The inventories of the parent revisions of the
354
            commit.
355
        :param path: The path the entry is at in the tree.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
356
        :param tree: The tree which contains this entry and should be used to
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
357
            obtain content.
358
        :param content_summary: Summary data from the tree about the paths
359
            content - stat, length, exec, sha/link target. This is only
360
            accessed when the entry has a revision of None - that is when it is
361
            a candidate to commit.
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
362
        :return: A tuple (change_delta, version_recorded, fs_hash).
363
            change_delta is an inventory_delta change for this entry against
364
            the basis tree of the commit, or None if no change occured against
365
            the basis tree.
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
366
            version_recorded is True if a new version of the entry has been
367
            recorded. For instance, committing a merge where a file was only
368
            changed on the other side will return (delta, False).
3709.3.3 by Robert Collins
NEWS for the record_entry_contents change.
369
            fs_hash is either None, or the hash details for the path (currently
370
            a tuple of the contents sha1 and the statvalue returned by
371
            tree.get_file_with_stat()).
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
372
        """
373
        if self.new_inventory.root is None:
2871.1.2 by Robert Collins
* ``CommitBuilder.record_entry_contents`` now requires the root entry of a
374
            if ie.parent_id is not None:
375
                raise errors.RootMissing()
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
376
            self._check_root(ie, parent_invs, tree)
377
        if ie.revision is None:
378
            kind = content_summary[0]
379
        else:
380
            # ie is carried over from a prior commit
381
            kind = ie.kind
382
        # XXX: repository specific check for nested tree support goes here - if
383
        # the repo doesn't want nested trees we skip it ?
384
        if (kind == 'tree-reference' and
385
            not self.repository._format.supports_tree_reference):
386
            # mismatch between commit builder logic and repository:
387
            # this needs the entry creation pushed down into the builder.
2776.4.18 by Robert Collins
Review feedback.
388
            raise NotImplementedError('Missing repository subtree support.')
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
389
        self.new_inventory.add(ie)
390
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
391
        # TODO: slow, take it out of the inner loop.
392
        try:
393
            basis_inv = parent_invs[0]
394
        except IndexError:
395
            basis_inv = Inventory(root_id=None)
396
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
397
        # ie.revision is always None if the InventoryEntry is considered
2776.4.13 by Robert Collins
Merge bzr.dev.
398
        # for committing. We may record the previous parents revision if the
399
        # content is actually unchanged against a sole head.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
400
        if ie.revision is not None:
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
401
            if not self._versioned_root and path == '':
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
402
                # repositories that do not version the root set the root's
3775.2.2 by Robert Collins
Teach CommitBuilder to accumulate inventory deltas.
403
                # revision to the new commit even when no change occurs (more
404
                # specifically, they do not record a revision on the root; and
405
                # the rev id is assigned to the root during deserialisation -
406
                # this masks when a change may have occurred against the basis.
407
                # To match this we always issue a delta, because the revision
408
                # of the root will always be changing.
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
409
                if ie.file_id in basis_inv:
410
                    delta = (basis_inv.id2path(ie.file_id), path,
411
                        ie.file_id, ie)
412
                else:
2871.1.3 by Robert Collins
* The CommitBuilder method ``record_entry_contents`` now returns summary
413
                    # add
414
                    delta = (None, path, ie.file_id, ie)
3879.2.3 by John Arbash Meinel
Hide the .basis_delta variable, and require callers to use .get_basis_delta()
415
                self._basis_delta.append(delta)
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
416
                return delta, False, None
2903.2.5 by Martin Pool
record_entry_contents should give back deltas for changed roots; clean it up a bit
417
            else:
418
                # we don't need to commit this, because the caller already
419
                # determined that an existing revision of this file is
3619.1.1 by Robert Collins
Tighten up the handling of carried-over inventory entries.
420
                # appropriate. If its not being considered for committing then
421
                # it and all its parents to the root must be unaltered so
422
                # no-change against the basis.
423
                if ie.revision == self._new_revision_id:
424
                    raise AssertionError("Impossible situation, a skipped "
3619.1.2 by Robert Collins
Review feedback.
425
                        "inventory entry (%r) claims to be modified in this "
426
                        "commit (%r).", (ie, self._new_revision_id))
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
427
                return None, False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
428
        # XXX: Friction: parent_candidates should return a list not a dict
429
        #      so that we don't have to walk the inventories again.
430
        parent_candiate_entries = ie.parent_candidates(parent_invs)
2979.2.5 by Robert Collins
Make CommitBuilder.heads be _heads as its internal to CommitBuilder only.
431
        head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
432
        heads = []
433
        for inv in parent_invs:
434
            if ie.file_id in inv:
435
                old_rev = inv[ie.file_id].revision
436
                if old_rev in head_set:
437
                    heads.append(inv[ie.file_id].revision)
438
                    head_set.remove(inv[ie.file_id].revision)
439
440
        store = False
441
        # now we check to see if we need to write a new record to the
442
        # file-graph.
443
        # We write a new entry unless there is one head to the ancestors, and
444
        # the kind-derived content is unchanged.
445
446
        # Cheapest check first: no ancestors, or more the one head in the
447
        # ancestors, we write a new node.
448
        if len(heads) != 1:
449
            store = True
450
        if not store:
451
            # There is a single head, look it up for comparison
452
            parent_entry = parent_candiate_entries[heads[0]]
453
            # if the non-content specific data has changed, we'll be writing a
454
            # node:
455
            if (parent_entry.parent_id != ie.parent_id or
456
                parent_entry.name != ie.name):
457
                store = True
458
        # now we need to do content specific checks:
459
        if not store:
460
            # if the kind changed the content obviously has
461
            if kind != parent_entry.kind:
462
                store = True
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
463
        # Stat cache fingerprint feedback for the caller - None as we usually
464
        # don't generate one.
465
        fingerprint = None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
466
        if kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
467
            if content_summary[2] is None:
468
                raise ValueError("Files must not have executable = None")
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
469
            if not store:
470
                if (# if the file length changed we have to store:
471
                    parent_entry.text_size != content_summary[1] or
472
                    # if the exec bit has changed we have to store:
473
                    parent_entry.executable != content_summary[2]):
474
                    store = True
475
                elif parent_entry.text_sha1 == content_summary[3]:
476
                    # all meta and content is unchanged (using a hash cache
477
                    # hit to check the sha)
478
                    ie.revision = parent_entry.revision
479
                    ie.text_size = parent_entry.text_size
480
                    ie.text_sha1 = parent_entry.text_sha1
481
                    ie.executable = parent_entry.executable
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
482
                    return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
483
                else:
484
                    # Either there is only a hash change(no hash cache entry,
485
                    # or same size content change), or there is no change on
486
                    # this file at all.
2776.4.19 by Robert Collins
Final review tweaks.
487
                    # Provide the parent's hash to the store layer, so that the
488
                    # content is unchanged we will not store a new node.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
489
                    nostore_sha = parent_entry.text_sha1
490
            if store:
2776.4.18 by Robert Collins
Review feedback.
491
                # We want to record a new node regardless of the presence or
492
                # absence of a content change in the file.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
493
                nostore_sha = None
2776.4.18 by Robert Collins
Review feedback.
494
            ie.executable = content_summary[2]
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
495
            file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
496
            try:
497
                lines = file_obj.readlines()
498
            finally:
499
                file_obj.close()
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
500
            try:
501
                ie.text_sha1, ie.text_size = self._add_text_to_weave(
502
                    ie.file_id, lines, heads, nostore_sha)
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
503
                # Let the caller know we generated a stat fingerprint.
504
                fingerprint = (ie.text_sha1, stat_value)
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
505
            except errors.ExistingContent:
2776.4.18 by Robert Collins
Review feedback.
506
                # Turns out that the file content was unchanged, and we were
507
                # only going to store a new node if it was changed. Carry over
508
                # the entry.
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
509
                ie.revision = parent_entry.revision
510
                ie.text_size = parent_entry.text_size
511
                ie.text_sha1 = parent_entry.text_sha1
512
                ie.executable = parent_entry.executable
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
513
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
514
        elif kind == 'directory':
515
            if not store:
516
                # all data is meta here, nothing specific to directory, so
517
                # carry over:
518
                ie.revision = parent_entry.revision
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
519
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
520
            lines = []
521
            self._add_text_to_weave(ie.file_id, lines, heads, None)
522
        elif kind == 'symlink':
523
            current_link_target = content_summary[3]
524
            if not store:
2776.4.18 by Robert Collins
Review feedback.
525
                # symlink target is not generic metadata, check if it has
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
526
                # changed.
527
                if current_link_target != parent_entry.symlink_target:
528
                    store = True
529
            if not store:
530
                # unchanged, carry over.
531
                ie.revision = parent_entry.revision
532
                ie.symlink_target = parent_entry.symlink_target
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
533
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
534
            ie.symlink_target = current_link_target
535
            lines = []
536
            self._add_text_to_weave(ie.file_id, lines, heads, None)
537
        elif kind == 'tree-reference':
538
            if not store:
539
                if content_summary[3] != parent_entry.reference_revision:
540
                    store = True
541
            if not store:
542
                # unchanged, carry over.
543
                ie.reference_revision = parent_entry.reference_revision
544
                ie.revision = parent_entry.revision
3709.3.1 by Robert Collins
First cut - make it work - at updating the tree stat cache during commit.
545
                return self._get_delta(ie, basis_inv, path), False, None
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
546
            ie.reference_revision = content_summary[3]
547
            lines = []
548
            self._add_text_to_weave(ie.file_id, lines, heads, None)
549
        else:
550
            raise NotImplementedError('unknown kind')
551
        ie.revision = self._new_revision_id
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
552
        self._any_changes = True
3709.3.2 by Robert Collins
Race-free stat-fingerprint updating during commit via a new method get_file_with_stat.
553
        return self._get_delta(ie, basis_inv, path), True, fingerprint
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
554
3775.2.30 by Robert Collins
Remove the basis_tree parameter to record_iter_changes.
555
    def record_iter_changes(self, tree, basis_revision_id, iter_changes,
556
        _entry_factory=entry_factory):
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
557
        """Record a new tree via iter_changes.
558
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
559
        :param tree: The tree to obtain text contents from for changed objects.
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
560
        :param basis_revision_id: The revision id of the tree the iter_changes
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
561
            has been generated against. Currently assumed to be the same
562
            as self.parents[0] - if it is not, errors may occur.
563
        :param iter_changes: An iter_changes iterator with the changes to apply
4183.5.5 by Robert Collins
Enable record_iter_changes for cases where it can work.
564
            to basis_revision_id. The iterator must not include any items with
565
            a current kind of None - missing items must be either filtered out
566
            or errored-on beefore record_iter_changes sees the item.
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
567
        :param _entry_factory: Private method to bind entry_factory locally for
568
            performance.
4183.5.4 by Robert Collins
Turn record_iter_changes into a generator to emit file system hashes.
569
        :return: A generator of (file_id, relpath, fs_hash) tuples for use with
570
            tree._observed_sha1.
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
571
        """
572
        # Create an inventory delta based on deltas between all the parents and
573
        # deltas between all the parent inventories. We use inventory delta's 
574
        # between the inventory objects because iter_changes masks
575
        # last-changed-field only changes.
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
576
        # Working data:
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
577
        # file_id -> change map, change is fileid, paths, changed, versioneds,
578
        # parents, names, kinds, executables
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
579
        merged_ids = {}
3775.2.32 by Robert Collins
Trivial review feedback.
580
        # {file_id -> revision_id -> inventory entry, for entries in parent
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
581
        # trees that are not parents[0]
582
        parent_entries = {}
4183.5.5 by Robert Collins
Enable record_iter_changes for cases where it can work.
583
        ghost_basis = False
584
        try:
585
            revtrees = list(self.repository.revision_trees(self.parents))
586
        except errors.NoSuchRevision:
587
            # one or more ghosts, slow path.
588
            revtrees = []
589
            for revision_id in self.parents:
590
                try:
591
                    revtrees.append(self.repository.revision_tree(revision_id))
592
                except errors.NoSuchRevision:
593
                    if not revtrees:
594
                        basis_revision_id = _mod_revision.NULL_REVISION
595
                        ghost_basis = True
596
                    revtrees.append(self.repository.revision_tree(
597
                        _mod_revision.NULL_REVISION))
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
598
        # The basis inventory from a repository 
599
        if revtrees:
600
            basis_inv = revtrees[0].inventory
601
        else:
602
            basis_inv = self.repository.revision_tree(
603
                _mod_revision.NULL_REVISION).inventory
3775.2.32 by Robert Collins
Trivial review feedback.
604
        if len(self.parents) > 0:
4183.5.5 by Robert Collins
Enable record_iter_changes for cases where it can work.
605
            if basis_revision_id != self.parents[0] and not ghost_basis:
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
606
                raise Exception(
607
                    "arbitrary basis parents not yet supported with merges")
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
608
            for revtree in revtrees[1:]:
3775.2.32 by Robert Collins
Trivial review feedback.
609
                for change in revtree.inventory._make_delta(basis_inv):
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
610
                    if change[1] is None:
3775.2.32 by Robert Collins
Trivial review feedback.
611
                        # Not present in this parent.
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
612
                        continue
613
                    if change[2] not in merged_ids:
614
                        if change[0] is not None:
4183.5.9 by Robert Collins
Fix creating new revisions of files when merging.
615
                            basis_entry = basis_inv[change[2]]
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
616
                            merged_ids[change[2]] = [
4183.5.9 by Robert Collins
Fix creating new revisions of files when merging.
617
                                # basis revid
618
                                basis_entry.revision,
619
                                # new tree revid
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
620
                                change[3].revision]
4183.5.9 by Robert Collins
Fix creating new revisions of files when merging.
621
                            parent_entries[change[2]] = {
622
                                # basis parent
623
                                basis_entry.revision:basis_entry,
624
                                # this parent 
625
                                change[3].revision:change[3],
626
                                }
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
627
                        else:
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
628
                            merged_ids[change[2]] = [change[3].revision]
4183.5.9 by Robert Collins
Fix creating new revisions of files when merging.
629
                            parent_entries[change[2]] = {change[3].revision:change[3]}
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
630
                    else:
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
631
                        merged_ids[change[2]].append(change[3].revision)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
632
                        parent_entries[change[2]][change[3].revision] = change[3]
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
633
        else:
634
            merged_ids = {}
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
635
        # Setup the changes from the tree:
3775.2.32 by Robert Collins
Trivial review feedback.
636
        # changes maps file_id -> (change, [parent revision_ids])
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
637
        changes= {}
638
        for change in iter_changes:
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
639
            # This probably looks up in basis_inv way to much.
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
640
            if change[1][0] is not None:
641
                head_candidate = [basis_inv[change[0]].revision]
642
            else:
643
                head_candidate = []
644
            changes[change[0]] = change, merged_ids.get(change[0],
645
                head_candidate)
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
646
        unchanged_merged = set(merged_ids) - set(changes)
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
647
        # Extend the changes dict with synthetic changes to record merges of
648
        # texts.
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
649
        for file_id in unchanged_merged:
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
650
            # Record a merged version of these items that did not change vs the
651
            # basis. This can be either identical parallel changes, or a revert
652
            # of a specific file after a merge. The recorded content will be
653
            # that of the current tree (which is the same as the basis), but
654
            # the per-file graph will reflect a merge.
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
655
            # NB:XXX: We are reconstructing path information we had, this
656
            # should be preserved instead.
657
            # inv delta  change: (file_id, (path_in_source, path_in_target),
658
            #   changed_content, versioned, parent, name, kind,
659
            #   executable)
4183.5.5 by Robert Collins
Enable record_iter_changes for cases where it can work.
660
            try:
661
                basis_entry = basis_inv[file_id]
662
            except errors.NoSuchId:
663
                # a change from basis->some_parents but file_id isn't in basis
664
                # so was new in the merge, which means it must have changed
665
                # from basis -> current, and as it hasn't the add was reverted
666
                # by the user. So we discard this change.
667
                pass
668
            else:
669
                change = (file_id,
670
                    (basis_inv.id2path(file_id), tree.id2path(file_id)),
671
                    False, (True, True),
672
                    (basis_entry.parent_id, basis_entry.parent_id),
673
                    (basis_entry.name, basis_entry.name),
674
                    (basis_entry.kind, basis_entry.kind),
675
                    (basis_entry.executable, basis_entry.executable))
676
                changes[file_id] = (change, merged_ids[file_id])
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
677
        # changes contains tuples with the change and a set of inventory
678
        # candidates for the file.
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
679
        # inv delta is:
680
        # old_path, new_path, file_id, new_inventory_entry
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
681
        seen_root = False # Is the root in the basis delta?
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
682
        inv_delta = self._basis_delta
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
683
        modified_rev = self._new_revision_id
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
684
        for change, head_candidates in changes.values():
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
685
            if change[3][1]: # versioned in target.
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
686
                # Several things may be happening here:
687
                # We may have a fork in the per-file graph
688
                #  - record a change with the content from tree
689
                # We may have a change against < all trees  
690
                #  - carry over the tree that hasn't changed
691
                # We may have a change against all trees
692
                #  - record the change with the content from tree
3775.2.11 by Robert Collins
CommitBuilder handles renamed directory and unmodified entries with single parents, for record_iter_changes.
693
                kind = change[6][1]
3775.2.12 by Robert Collins
CommitBuilder.record_iter_changes handles renamed files.
694
                file_id = change[0]
695
                entry = _entry_factory[kind](file_id, change[5][1],
696
                    change[4][1])
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
697
                head_set = self._heads(change[0], set(head_candidates))
698
                heads = []
699
                # Preserve ordering.
700
                for head_candidate in head_candidates:
701
                    if head_candidate in head_set:
702
                        heads.append(head_candidate)
703
                        head_set.remove(head_candidate)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
704
                carried_over = False
3775.2.33 by Robert Collins
Fix bug with merges of new files, increasing test coverage to ensure its kept fixed.
705
                if len(heads) == 1:
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
706
                    # Could be a carry-over situation:
3775.2.34 by Robert Collins
Handle committing new files again.
707
                    parent_entry_revs = parent_entries.get(file_id, None)
708
                    if parent_entry_revs:
709
                        parent_entry = parent_entry_revs.get(heads[0], None)
710
                    else:
711
                        parent_entry = None
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
712
                    if parent_entry is None:
713
                        # The parent iter_changes was called against is the one
714
                        # that is the per-file head, so any change is relevant
715
                        # iter_changes is valid.
716
                        carry_over_possible = False
717
                    else:
718
                        # could be a carry over situation
719
                        # A change against the basis may just indicate a merge,
720
                        # we need to check the content against the source of the
721
                        # merge to determine if it was changed after the merge
722
                        # or carried over.
3775.2.23 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch files.
723
                        if (parent_entry.kind != entry.kind or
724
                            parent_entry.parent_id != entry.parent_id or
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
725
                            parent_entry.name != entry.name):
726
                            # Metadata common to all entries has changed
727
                            # against per-file parent
728
                            carry_over_possible = False
729
                        else:
730
                            carry_over_possible = True
731
                        # per-type checks for changes against the parent_entry
732
                        # are done below.
733
                else:
734
                    # Cannot be a carry-over situation
735
                    carry_over_possible = False
736
                # Populate the entry in the delta
737
                if kind == 'file':
3775.2.23 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch files.
738
                    # XXX: There is still a small race here: If someone reverts the content of a file
739
                    # after iter_changes examines and decides it has changed,
740
                    # we will unconditionally record a new version even if some
741
                    # other process reverts it while commit is running (with
742
                    # the revert happening after iter_changes did it's
743
                    # examination).
744
                    if change[7][1]:
745
                        entry.executable = True
746
                    else:
747
                        entry.executable = False
748
                    if (carry_over_possible and 
749
                        parent_entry.executable == entry.executable):
750
                            # Check the file length, content hash after reading
751
                            # the file.
752
                            nostore_sha = parent_entry.text_sha1
753
                    else:
754
                        nostore_sha = None
755
                    file_obj, stat_value = tree.get_file_with_stat(file_id, change[1][1])
756
                    try:
757
                        lines = file_obj.readlines()
758
                    finally:
759
                        file_obj.close()
760
                    try:
761
                        entry.text_sha1, entry.text_size = self._add_text_to_weave(
762
                            file_id, lines, heads, nostore_sha)
4183.5.4 by Robert Collins
Turn record_iter_changes into a generator to emit file system hashes.
763
                        yield file_id, change[1][1], (entry.text_sha1, stat_value)
3775.2.23 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch files.
764
                    except errors.ExistingContent:
765
                        # No content change against a carry_over parent
4183.5.4 by Robert Collins
Turn record_iter_changes into a generator to emit file system hashes.
766
                        # Perhaps this should also yield a fs hash update?
3775.2.23 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch files.
767
                        carried_over = True
768
                        entry.text_size = parent_entry.text_size
769
                        entry.text_sha1 = parent_entry.text_sha1
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
770
                elif kind == 'symlink':
3775.2.24 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch symlinks.
771
                    # Wants a path hint?
772
                    entry.symlink_target = tree.get_symlink_target(file_id)
773
                    if (carry_over_possible and
774
                        parent_entry.symlink_target == entry.symlink_target):
4183.5.2 by Robert Collins
Support tree-reference in record_iter_changes.
775
                        carried_over = True
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
776
                    else:
3775.2.13 by Robert Collins
CommitBuilder.record_iter_changes handles renamed symlinks.
777
                        self._add_text_to_weave(change[0], [], heads, None)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
778
                elif kind == 'directory':
779
                    if carry_over_possible:
780
                        carried_over = True
781
                    else:
3775.2.19 by Robert Collins
CommitBuilder.record_iter_changes handles merged directories.
782
                        # Nothing to set on the entry.
783
                        # XXX: split into the Root and nonRoot versions.
784
                        if change[1][1] != '' or self.repository.supports_rich_root():
785
                            self._add_text_to_weave(change[0], [], heads, None)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
786
                elif kind == 'tree-reference':
4183.5.2 by Robert Collins
Support tree-reference in record_iter_changes.
787
                    if not self.repository._format.supports_tree_reference:
788
                        # This isn't quite sane as an error, but we shouldn't
789
                        # ever see this code path in practice: tree's don't
790
                        # permit references when the repo doesn't support tree
791
                        # references.
792
                        raise errors.UnsupportedOperation(tree.add_reference,
793
                            self.repository)
794
                    entry.reference_revision = \
795
                        tree.get_reference_revision(change[0])
796
                    if (carry_over_possible and
797
                        parent_entry.reference_revision == reference_revision):
798
                        carried_over = True
799
                    else:
800
                        self._add_text_to_weave(change[0], [], heads, None)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
801
                else:
3775.2.27 by Robert Collins
CommitBuilder.record_iter_changes handles files becoming directories and links.
802
                    raise AssertionError('unknown kind %r' % kind)
3775.2.22 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch directories.
803
                if not carried_over:
804
                    entry.revision = modified_rev
3775.2.23 by Robert Collins
CommitBuilder.record_iter_changes handles changed-in-branch files.
805
                else:
806
                    entry.revision = parent_entry.revision
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
807
            else:
808
                entry = None
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
809
            new_path = change[1][1]
810
            inv_delta.append((change[1][0], new_path, change[0], entry))
811
            if new_path == '':
812
                seen_root = True
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
813
        self.new_inventory = None
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
814
        if len(inv_delta):
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
815
            self._any_changes = True
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
816
        if not seen_root:
817
            # housekeeping root entry changes do not affect no-change commits.
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
818
            self._require_root_change(tree)
3775.2.29 by Robert Collins
Updates to the form of add_inventory_by_delta that landed in trunk.
819
        self.basis_delta_revision = basis_revision_id
3775.2.4 by Robert Collins
Start on a CommitBuilder.record_iter_changes method.
820
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
821
    def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
2592.3.136 by Robert Collins
Merge bzr.dev.
822
        # Note: as we read the content directly from the tree, we know its not
823
        # been turned into unicode or badly split - but a broken tree
824
        # implementation could give us bad output from readlines() so this is
825
        # not a guarantee of safety. What would be better is always checking
826
        # the content during test suite execution. RBC 20070912
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
827
        parent_keys = tuple((file_id, parent) for parent in parents)
828
        return self.repository.texts.add_lines(
829
            (file_id, self._new_revision_id), parent_keys, new_lines,
3316.2.12 by Robert Collins
Catch some extra deprecated calls.
830
            nostore_sha=nostore_sha, random_id=self.random_revid,
831
            check_content=False)[0:2]
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
832
833
834
class RootCommitBuilder(CommitBuilder):
835
    """This commitbuilder actually records the root id"""
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
836
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
837
    # the root entry gets versioned properly by this builder.
2840.1.1 by Ian Clatworthy
faster pointless commit detection (Robert Collins)
838
    _versioned_root = True
2825.5.2 by Robert Collins
Review feedback, and fix pointless commits with nested trees to raise PointlessCommit appropriately.
839
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
840
    def _check_root(self, ie, parent_invs, tree):
841
        """Helper for record_entry_contents.
842
843
        :param ie: An entry being added.
844
        :param parent_invs: The inventories of the parent revisions of the
845
            commit.
846
        :param tree: The tree that is being committed.
847
        """
848
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
849
    def _require_root_change(self, tree):
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
850
        """Enforce an appropriate root object change.
851
852
        This is called once when record_iter_changes is called, if and only if
853
        the root was not in the delta calculated by record_iter_changes.
3775.2.9 by Robert Collins
CommitBuilder handles deletes via record_iter_entries.
854
855
        :param tree: The tree which is being committed.
3775.2.7 by Robert Collins
CommitBuilder handles no-change commits to roots properly with record_iter_changes.
856
        """
857
        # versioned roots do not change unless the tree found a change.
858
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
859
2220.2.3 by Martin Pool
Add tag: revision namespace.
860
######################################################################
861
# Repositories
862
1185.66.5 by Aaron Bentley
Renamed RevisionStorage to Repository
863
class Repository(object):
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
864
    """Repository holding history for one or more branches.
865
866
    The repository holds and retrieves historical information including
867
    revisions and file history.  It's normally accessed only by the Branch,
868
    which views a particular line of development through that history.
869
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
870
    The Repository builds on top of some byte storage facilies (the revisions,
3735.2.1 by Robert Collins
Add the concept of CHK lookups to Repository.
871
    signatures, inventories, texts and chk_bytes attributes) and a Transport,
872
    which respectively provide byte storage and a means to access the (possibly
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
873
    remote) disk.
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
874
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
875
    The byte storage facilities are addressed via tuples, which we refer to
876
    as 'keys' throughout the code base. Revision_keys, inventory_keys and
877
    signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
3735.2.1 by Robert Collins
Add the concept of CHK lookups to Repository.
878
    (file_id, revision_id). chk_bytes uses CHK keys - a 1-tuple with a single
3735.2.99 by John Arbash Meinel
Merge bzr.dev 4034. Whitespace cleanup
879
    byte string made up of a hash identifier and a hash value.
3735.2.1 by Robert Collins
Add the concept of CHK lookups to Repository.
880
    We use this interface because it allows low friction with the underlying
881
    code that implements disk indices, network encoding and other parts of
882
    bzrlib.
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
883
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
884
    :ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
885
        the serialised revisions for the repository. This can be used to obtain
886
        revision graph information or to access raw serialised revisions.
887
        The result of trying to insert data into the repository via this store
888
        is undefined: it should be considered read-only except for implementors
889
        of repositories.
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
890
    :ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
891
        the serialised signatures for the repository. This can be used to
892
        obtain access to raw serialised signatures.  The result of trying to
893
        insert data into the repository via this store is undefined: it should
894
        be considered read-only except for implementors of repositories.
895
    :ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
896
        the serialised inventories for the repository. This can be used to
897
        obtain unserialised inventories.  The result of trying to insert data
898
        into the repository via this store is undefined: it should be
899
        considered read-only except for implementors of repositories.
900
    :ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
901
        texts of files and directories for the repository. This can be used to
902
        obtain file texts or file graphs. Note that Repository.iter_file_bytes
903
        is usually a better interface for accessing file texts.
904
        The result of trying to insert data into the repository via this store
905
        is undefined: it should be considered read-only except for implementors
906
        of repositories.
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
907
    :ivar chk_bytes: A bzrlib.versionedfile.VersionedFiles instance containing
3735.2.1 by Robert Collins
Add the concept of CHK lookups to Repository.
908
        any data the repository chooses to store or have indexed by its hash.
909
        The result of trying to insert data into the repository via this store
910
        is undefined: it should be considered read-only except for implementors
911
        of repositories.
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
912
    :ivar _transport: Transport for file access to repository, typically
913
        pointing to .bzr/repository.
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
914
    """
1185.65.17 by Robert Collins
Merge from integration, mode-changes are broken.
915
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
916
    # What class to use for a CommitBuilder. Often its simpler to change this
917
    # in a Repository class subclass rather than to override
918
    # get_commit_builder.
919
    _commit_builder_class = CommitBuilder
920
    # The search regex used by xml based repositories to determine what things
921
    # where changed in a single commit.
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
922
    _file_ids_altered_regex = lazy_regex.lazy_compile(
923
        r'file_id="(?P<file_id>[^"]+)"'
2776.4.6 by Robert Collins
Fixup various commit test failures falling out from the other commit changes.
924
        r'.* revision="(?P<revision_id>[^"]+)"'
2163.2.1 by John Arbash Meinel
Speed up the fileids_altered_by_revision_ids processing
925
        )
926
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
927
    def abort_write_group(self, suppress_errors=False):
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
928
        """Commit the contents accrued within the current write group.
929
3825.4.6 by Andrew Bennetts
Document the suppress_errors flag in the docstring.
930
        :param suppress_errors: if true, abort_write_group will catch and log
931
            unexpected errors that happen during the abort, rather than
932
            allowing them to propagate.  Defaults to False.
933
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
934
        :seealso: start_write_group.
935
        """
936
        if self._write_group is not self.get_transaction():
937
            # has an unlock or relock occured ?
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
938
            raise errors.BzrError(
939
                'mismatched lock context and write group. %r, %r' %
940
                (self._write_group, self.get_transaction()))
3825.4.1 by Andrew Bennetts
Add suppress_errors to abort_write_group.
941
        try:
942
            self._abort_write_group()
943
        except Exception, exc:
944
            self._write_group = None
945
            if not suppress_errors:
946
                raise
947
            mutter('abort_write_group failed')
948
            log_exception_quietly()
949
            note('bzr: ERROR (ignored): %s', exc)
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
950
        self._write_group = None
951
952
    def _abort_write_group(self):
953
        """Template method for per-repository write group cleanup.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
954
955
        This is called during abort before the write group is considered to be
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
956
        finished and should cleanup any internal state accrued during the write
957
        group. There is no requirement that data handed to the repository be
958
        *not* made available - this is not a rollback - but neither should any
959
        attempt be made to ensure that data added is fully commited. Abort is
960
        invoked when an error has occured so futher disk or network operations
961
        may not be possible or may error and if possible should not be
962
        attempted.
963
        """
964
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
965
    def add_fallback_repository(self, repository):
966
        """Add a repository to use for looking up data not held locally.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
967
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
968
        :param repository: A repository.
969
        """
970
        if not self._format.supports_external_lookups:
971
            raise errors.UnstackableRepositoryFormat(self._format, self.base)
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
972
        self._check_fallback_repository(repository)
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
973
        self._fallback_repositories.append(repository)
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
974
        self.texts.add_fallback_versioned_files(repository.texts)
975
        self.inventories.add_fallback_versioned_files(repository.inventories)
976
        self.revisions.add_fallback_versioned_files(repository.revisions)
977
        self.signatures.add_fallback_versioned_files(repository.signatures)
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
978
        if self.chk_bytes is not None:
979
            self.chk_bytes.add_fallback_versioned_files(repository.chk_bytes)
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
980
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
981
    def _check_fallback_repository(self, repository):
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
982
        """Check that this repository can fallback to repository safely.
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
983
984
        Raise an error if not.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
985
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
986
        :param repository: A repository to fallback to.
987
        """
3582.1.7 by Martin Pool
add_fallback_repository gives more detail on incompatibilities
988
        return InterRepository._assert_same_model(self, repository)
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
989
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
990
    def add_inventory(self, revision_id, inv, parents):
991
        """Add the inventory inv to the repository as revision_id.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
992
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
993
        :param parents: The revision ids of the parents that revision_id
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
994
                        is known to have and are in the repository already.
995
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
996
        :returns: The validator(which is a sha1 digest, though what is sha'd is
997
            repository format specific) of the serialized inventory.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
998
        """
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
999
        if not self.is_in_write_group():
1000
            raise AssertionError("%r not in write group" % (self,))
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1001
        _mod_revision.check_not_reserved_id(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1002
        if not (inv.revision_id is None or inv.revision_id == revision_id):
1003
            raise AssertionError(
1004
                "Mismatch between inventory revision"
1005
                " id and insertion revid (%r, %r)"
1006
                % (inv.revision_id, revision_id))
1007
        if inv.root is None:
1008
            raise AssertionError()
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
1009
        return self._add_inventory_checked(revision_id, inv, parents)
1010
1011
    def _add_inventory_checked(self, revision_id, inv, parents):
1012
        """Add inv to the repository after checking the inputs.
1013
1014
        This function can be overridden to allow different inventory styles.
1015
1016
        :seealso: add_inventory, for the contract.
1017
        """
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
1018
        inv_lines = self._serialise_inventory_to_lines(inv)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1019
        return self._inventory_add_lines(revision_id, parents,
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
1020
            inv_lines, check_content=False)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1021
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
1022
    def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
3735.2.121 by Ian Clatworthy
add propagate_caches param to create_by_apply_delta, making fast-import 30% faster
1023
                               parents, basis_inv=None, propagate_caches=False):
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1024
        """Add a new inventory expressed as a delta against another revision.
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
1025
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1026
        :param basis_revision_id: The inventory id the delta was created
3879.2.2 by John Arbash Meinel
Rename add_inventory_delta to add_inventory_by_delta.
1027
            against. (This does not have to be a direct parent.)
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1028
        :param delta: The inventory delta (see Inventory.apply_delta for
1029
            details).
1030
        :param new_revision_id: The revision id that the inventory is being
1031
            added for.
1032
        :param parents: The revision ids of the parents that revision_id is
1033
            known to have and are in the repository already. These are supplied
1034
            for repositories that depend on the inventory graph for revision
1035
            graph access, as well as for those that pun ancestry with delta
1036
            compression.
3735.2.120 by Ian Clatworthy
allow a known basis inventory to be passed to Repository.add_inventory_by_delta()
1037
        :param basis_inv: The basis inventory if it is already known,
1038
            otherwise None.
3735.2.121 by Ian Clatworthy
add propagate_caches param to create_by_apply_delta, making fast-import 30% faster
1039
        :param propagate_caches: If True, the caches for this inventory are
1040
          copied to and updated for the result if possible.
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1041
3879.3.1 by John Arbash Meinel
Change the return of add_inventory_by_delta to also return the Inventory.
1042
        :returns: (validator, new_inv)
1043
            The validator(which is a sha1 digest, though what is sha'd is
1044
            repository format specific) of the serialized inventory, and the
1045
            resulting inventory.
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1046
        """
1047
        if not self.is_in_write_group():
1048
            raise AssertionError("%r not in write group" % (self,))
1049
        _mod_revision.check_not_reserved_id(new_revision_id)
1050
        basis_tree = self.revision_tree(basis_revision_id)
1051
        basis_tree.lock_read()
1052
        try:
1053
            # Note that this mutates the inventory of basis_tree, which not all
1054
            # inventory implementations may support: A better idiom would be to
1055
            # return a new inventory, but as there is no revision tree cache in
1056
            # repository this is safe for now - RBC 20081013
3735.2.120 by Ian Clatworthy
allow a known basis inventory to be passed to Repository.add_inventory_by_delta()
1057
            if basis_inv is None:
1058
                basis_inv = basis_tree.inventory
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1059
            basis_inv.apply_delta(delta)
1060
            basis_inv.revision_id = new_revision_id
3879.3.1 by John Arbash Meinel
Change the return of add_inventory_by_delta to also return the Inventory.
1061
            return (self.add_inventory(new_revision_id, basis_inv, parents),
3735.2.59 by Jelmer Vernooij
Make Repository.add_inventory_delta() return the resulting inventory.
1062
                    basis_inv)
3775.2.1 by Robert Collins
Create bzrlib.repository.Repository.add_inventory_delta for adding inventories via deltas.
1063
        finally:
1064
            basis_tree.unlock()
1065
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1066
    def _inventory_add_lines(self, revision_id, parents, lines,
2805.6.7 by Robert Collins
Review feedback.
1067
        check_content=True):
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
1068
        """Store lines in inv_vf and return the sha1 of the inventory."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1069
        parents = [(parent,) for parent in parents]
1070
        return self.inventories.add_lines((revision_id,), parents, lines,
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
1071
            check_content=check_content)[0]
1740.3.6 by Jelmer Vernooij
Move inventory writing to the commit builder.
1072
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1073
    def add_revision(self, revision_id, rev, inv=None, config=None):
1074
        """Add rev to the revision store as revision_id.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1075
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1076
        :param revision_id: the revision id to use.
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1077
        :param rev: The revision object.
1078
        :param inv: The inventory for the revision. if None, it will be looked
1079
                    up in the inventory storer
1080
        :param config: If None no digital signature will be created.
1081
                       If supplied its signature_needed method will be used
1082
                       to determine if a signature should be made.
1083
        """
2249.5.13 by John Arbash Meinel
Finish auditing Repository, and fix generate_ids to always generate utf8 ids.
1084
        # TODO: jam 20070210 Shouldn't we check rev.revision_id and
1085
        #       rev.parent_ids?
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1086
        _mod_revision.check_not_reserved_id(revision_id)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1087
        if config is not None and config.signature_needed():
1088
            if inv is None:
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1089
                inv = self.get_inventory(revision_id)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1090
            plaintext = Testament(rev, inv).as_short_text()
1091
            self.store_revision_signature(
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1092
                gpg.GPGStrategy(config), plaintext, revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1093
        # check inventory present
1094
        if not self.inventories.get_parent_map([(revision_id,)]):
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1095
            if inv is None:
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
1096
                raise errors.WeaveRevisionNotPresent(revision_id,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1097
                                                     self.inventories)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1098
            else:
1099
                # yes, this is not suitable for adding with ghosts.
3380.1.6 by Aaron Bentley
Ensure fetching munges sha1s
1100
                rev.inventory_sha1 = self.add_inventory(revision_id, inv,
3305.1.1 by Jelmer Vernooij
Make sure that specifying the inv= argument to add_revision() sets the
1101
                                                        rev.parent_ids)
3380.1.6 by Aaron Bentley
Ensure fetching munges sha1s
1102
        else:
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1103
            key = (revision_id,)
1104
            rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1105
        self._add_revision(rev)
1570.1.2 by Robert Collins
Import bzrtools' 'fix' command as 'bzr reconcile.'
1106
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1107
    def _add_revision(self, revision):
1108
        text = self._serializer.write_revision_to_string(revision)
1109
        key = (revision.revision_id,)
1110
        parents = tuple((parent,) for parent in revision.parent_ids)
1111
        self.revisions.add_lines(key, parents, osutils.split_lines(text))
2520.4.10 by Aaron Bentley
Enable installation of revisions
1112
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1113
    def all_revision_ids(self):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1114
        """Returns a list of all the revision ids in the repository.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1115
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
1116
        This is conceptually deprecated because code should generally work on
1117
        the graph reachable from a particular revision, and ignore any other
1118
        revisions that might be present.  There is no direct replacement
1119
        method.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1120
        """
2592.3.114 by Robert Collins
More evil mutterings.
1121
        if 'evil' in debug.debug_flags:
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1122
            mutter_callsite(2, "all_revision_ids is linear with history.")
3221.12.4 by Robert Collins
Implement basic repository supporting external references.
1123
        return self._all_revision_ids()
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
1124
1125
    def _all_revision_ids(self):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1126
        """Returns a list of all the revision ids in the repository.
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1127
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1128
        These are in as much topological order as the underlying store can
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1129
        present.
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1130
        """
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1131
        raise NotImplementedError(self._all_revision_ids)
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1132
1687.1.7 by Robert Collins
Teach Repository about break_lock.
1133
    def break_lock(self):
1134
        """Break a lock if one is present from another instance.
1135
1136
        Uses the ui factory to ask for confirmation if the lock may be from
1137
        an active process.
1138
        """
1139
        self.control_files.break_lock()
1140
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1141
    @needs_read_lock
1142
    def _eliminate_revisions_not_present(self, revision_ids):
1143
        """Check every revision id in revision_ids to see if we have it.
1144
1145
        Returns a set of the present revisions.
1146
        """
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1147
        result = []
3369.2.1 by John Arbash Meinel
Knit => knit fetching also has some very bad 'for x in revision_ids: has_revision_id()' calls
1148
        graph = self.get_graph()
1149
        parent_map = graph.get_parent_map(revision_ids)
1150
        # The old API returned a list, should this actually be a set?
1151
        return parent_map.keys()
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1152
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1153
    @staticmethod
1154
    def create(a_bzrdir):
1155
        """Construct the current default format repository in a_bzrdir."""
1156
        return RepositoryFormat.get_default_format().initialize(a_bzrdir)
1157
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1158
    def __init__(self, _format, a_bzrdir, control_files):
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
1159
        """instantiate a Repository.
1160
1161
        :param _format: The format of the repository on disk.
1162
        :param a_bzrdir: The BzrDir of the repository.
1163
1164
        In the future we will have a single api for all stores for
1165
        getting file texts, inventories and revisions, then
1166
        this construct will accept instances of those things.
1167
        """
1608.2.1 by Martin Pool
[merge] Storage filename escaping
1168
        super(Repository, self).__init__()
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1169
        self._format = _format
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
1170
        # the following are part of the public API for Repository:
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1171
        self.bzrdir = a_bzrdir
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
1172
        self.control_files = control_files
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
1173
        self._transport = control_files._transport
3407.2.14 by Martin Pool
Remove more cases of getting transport via control_files
1174
        self.base = self._transport.base
2671.4.2 by Robert Collins
Review feedback.
1175
        # for tests
1176
        self._reconcile_does_inventory_gc = True
2745.6.16 by Aaron Bentley
Update from review
1177
        self._reconcile_fixes_text_parents = False
2951.1.3 by Robert Collins
Partial support for native reconcile with packs.
1178
        self._reconcile_backsup_inventory = True
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1179
        # not right yet - should be more semantically clear ?
1180
        #
1608.2.1 by Martin Pool
[merge] Storage filename escaping
1181
        # TODO: make sure to construct the right store classes, etc, depending
1182
        # on whether escaping is required.
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
1183
        self._warn_if_deprecated()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1184
        self._write_group = None
3221.12.1 by Robert Collins
Backport development1 format (stackable packs) to before-shallow-branches.
1185
        # Additional places to query for data.
1186
        self._fallback_repositories = []
3882.6.23 by John Arbash Meinel
Change the XMLSerializer.read_inventory_from_string api.
1187
        # An InventoryEntry cache, used during deserialization
1188
        self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1189
1668.1.3 by Martin Pool
[patch] use the correct transaction when committing snapshot (Malone: #43959)
1190
    def __repr__(self):
2592.4.5 by Martin Pool
Add Repository.base on all repositories.
1191
        return '%s(%r)' % (self.__class__.__name__,
1192
                           self.base)
1668.1.3 by Martin Pool
[patch] use the correct transaction when committing snapshot (Malone: #43959)
1193
2671.1.4 by Andrew Bennetts
Rename is_same_repository to has_same_location, thanks Aaron!
1194
    def has_same_location(self, other):
2671.1.3 by Andrew Bennetts
Remove Repository.__eq__/__ne__ methods, replace with is_same_repository method.
1195
        """Returns a boolean indicating if this repository is at the same
1196
        location as another repository.
1197
1198
        This might return False even when two repository objects are accessing
1199
        the same physical repository via different URLs.
1200
        """
2592.3.162 by Robert Collins
Remove some arbitrary differences from bzr.dev.
1201
        if self.__class__ is not other.__class__:
1202
            return False
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
1203
        return (self._transport.base == other._transport.base)
2671.1.1 by Andrew Bennetts
Add support for comparing Repositories with == and != operators.
1204
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1205
    def is_in_write_group(self):
1206
        """Return True if there is an open write group.
1207
1208
        :seealso: start_write_group.
1209
        """
1210
        return self._write_group is not None
1211
1694.2.6 by Martin Pool
[merge] bzr.dev
1212
    def is_locked(self):
1213
        return self.control_files.is_locked()
1214
2592.3.188 by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency.
1215
    def is_write_locked(self):
1216
        """Return True if this object is write locked."""
1217
        return self.is_locked() and self.control_files._lock_mode == 'w'
1218
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1219
    def lock_write(self, token=None):
1220
        """Lock this repository for writing.
2617.6.8 by Robert Collins
Review feedback and documentation.
1221
1222
        This causes caching within the repository obejct to start accumlating
1223
        data during reads, and allows a 'write_group' to be obtained. Write
1224
        groups must be used for actual data insertion.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1225
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1226
        :param token: if this is already locked, then lock_write will fail
1227
            unless the token matches the existing lock.
1228
        :returns: a token if this instance supports tokens, otherwise None.
1229
        :raises TokenLockingNotSupported: when a token is given but this
1230
            instance doesn't support using token locks.
1231
        :raises MismatchedToken: if the specified token doesn't match the token
1232
            of the existing lock.
2617.6.8 by Robert Collins
Review feedback and documentation.
1233
        :seealso: start_write_group.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1234
2018.5.145 by Andrew Bennetts
Add a brief explanation of what tokens are used for to lock_write docstrings.
1235
        A token should be passed in if you know that you have locked the object
1236
        some other way, and need to synchronise this object's state with that
1237
        fact.
1238
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1239
        XXX: this docstring is duplicated in many places, e.g. lockable_files.py
1240
        """
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
1241
        locked = self.is_locked()
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1242
        result = self.control_files.lock_write(token=token)
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
1243
        for repo in self._fallback_repositories:
1244
            # Writes don't affect fallback repos
1245
            repo.lock_read()
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
1246
        if not locked:
1247
            self._refresh_data()
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1248
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1249
1250
    def lock_read(self):
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
1251
        locked = self.is_locked()
1553.5.55 by Martin Pool
[revert] broken changes
1252
        self.control_files.lock_read()
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
1253
        for repo in self._fallback_repositories:
1254
            repo.lock_read()
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
1255
        if not locked:
1256
            self._refresh_data()
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1257
1694.2.6 by Martin Pool
[merge] bzr.dev
1258
    def get_physical_lock_status(self):
1259
        return self.control_files.get_physical_lock_status()
1624.3.36 by Olaf Conradi
Rename is_transport_locked() to get_physical_lock_status() as the
1260
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1261
    def leave_lock_in_place(self):
1262
        """Tell this repository not to release the physical lock when this
1263
        object is unlocked.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1264
2018.5.76 by Andrew Bennetts
Testing that repository.{dont_,}leave_lock_in_place raises NotImplementedError if lock_write returns None.
1265
        If lock_write doesn't return a token, then this method is not supported.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1266
        """
1267
        self.control_files.leave_in_place()
1268
1269
    def dont_leave_lock_in_place(self):
1270
        """Tell this repository to release the physical lock when this
1271
        object is unlocked, even if it didn't originally acquire it.
2018.5.76 by Andrew Bennetts
Testing that repository.{dont_,}leave_lock_in_place raises NotImplementedError if lock_write returns None.
1272
1273
        If lock_write doesn't return a token, then this method is not supported.
2018.5.75 by Andrew Bennetts
Add Repository.{dont_,}leave_lock_in_place.
1274
        """
1275
        self.control_files.dont_leave_in_place()
1276
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1277
    @needs_read_lock
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1278
    def gather_stats(self, revid=None, committers=None):
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1279
        """Gather statistics from a revision id.
1280
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1281
        :param revid: The revision id to gather statistics from, if None, then
1282
            no revision specific statistics are gathered.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1283
        :param committers: Optional parameter controlling whether to grab
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1284
            a count of committers from the revision specific statistics.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1285
        :return: A dictionary of statistics. Currently this contains:
1286
            committers: The number of committers if requested.
1287
            firstrev: A tuple with timestamp, timezone for the penultimate left
1288
                most ancestor of revid, if revid is not the NULL_REVISION.
1289
            latestrev: A tuple with timestamp, timezone for revid, if revid is
1290
                not the NULL_REVISION.
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1291
            revisions: The total revision count in the repository.
1292
            size: An estimate disk size of the repository in bytes.
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1293
        """
1294
        result = {}
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1295
        if revid and committers:
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1296
            result['committers'] = 0
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1297
        if revid and revid != _mod_revision.NULL_REVISION:
1298
            if committers:
1299
                all_committers = set()
1300
            revisions = self.get_ancestry(revid)
1301
            # pop the leading None
1302
            revisions.pop(0)
1303
            first_revision = None
1304
            if not committers:
1305
                # ignore the revisions in the middle - just grab first and last
1306
                revisions = revisions[0], revisions[-1]
1307
            for revision in self.get_revisions(revisions):
1308
                if not first_revision:
1309
                    first_revision = revision
1310
                if committers:
1311
                    all_committers.add(revision.committer)
1312
            last_revision = revision
1313
            if committers:
1314
                result['committers'] = len(all_committers)
1315
            result['firstrev'] = (first_revision.timestamp,
1316
                first_revision.timezone)
1317
            result['latestrev'] = (last_revision.timestamp,
1318
                last_revision.timezone)
1319
1320
        # now gather global repository information
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1321
        # XXX: This is available for many repos regardless of listability.
2258.1.2 by Robert Collins
New version of gather_stats which gathers aggregate data too.
1322
        if self.bzrdir.root_transport.listable():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1323
            # XXX: do we want to __define len__() ?
3350.6.10 by Martin Pool
VersionedFiles review cleanups
1324
            # Maybe the versionedfiles object should provide a different
1325
            # method to get the number of keys.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1326
            result['revisions'] = len(self.revisions.keys())
1327
            # result['size'] = t
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1328
        return result
1329
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
1330
    def find_branches(self, using=False):
1331
        """Find branches underneath this repository.
1332
3140.1.7 by Aaron Bentley
Update docs
1333
        This will include branches inside other branches.
1334
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
1335
        :param using: If True, list only branches using this repository.
1336
        """
3140.1.9 by Aaron Bentley
Optimize find_branches for standalone repositories
1337
        if using and not self.is_shared():
1338
            try:
1339
                return [self.bzrdir.open_branch()]
1340
            except errors.NotBranchError:
1341
                return []
3140.1.2 by Aaron Bentley
Add ability to find branches inside repositories
1342
        class Evaluator(object):
1343
1344
            def __init__(self):
1345
                self.first_call = True
1346
1347
            def __call__(self, bzrdir):
1348
                # On the first call, the parameter is always the bzrdir
1349
                # containing the current repo.
1350
                if not self.first_call:
1351
                    try:
1352
                        repository = bzrdir.open_repository()
1353
                    except errors.NoRepositoryPresent:
1354
                        pass
1355
                    else:
1356
                        return False, (None, repository)
1357
                self.first_call = False
1358
                try:
1359
                    value = (bzrdir.open_branch(), None)
1360
                except errors.NotBranchError:
1361
                    value = (None, None)
1362
                return True, value
1363
1364
        branches = []
1365
        for branch, repository in bzrdir.BzrDir.find_bzrdirs(
1366
                self.bzrdir.root_transport, evaluate=Evaluator()):
1367
            if branch is not None:
1368
                branches.append(branch)
1369
            if not using and repository is not None:
1370
                branches.extend(repository.find_branches())
1371
        return branches
1372
2258.1.1 by Robert Collins
Move info branch statistics gathering into the repository to allow smart server optimisation (Robert Collins).
1373
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
1374
    def search_missing_revision_ids(self, other, revision_id=None, find_ghosts=True):
1375
        """Return the revision ids that other has that this does not.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1376
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
1377
        These are returned in topological order.
1378
1379
        revision_id: only return revision ids included by revision_id.
1380
        """
1381
        return InterRepository.get(other, self).search_missing_revision_ids(
1382
            revision_id, find_ghosts)
1383
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1384
    @staticmethod
1385
    def open(base):
1386
        """Open the repository rooted at base.
1387
1388
        For instance, if the repository is at URL/.bzr/repository,
1389
        Repository.open(URL) -> a Repository instance.
1390
        """
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
1391
        control = bzrdir.BzrDir.open(base)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
1392
        return control.open_repository()
1393
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1394
    def copy_content_into(self, destination, revision_id=None):
1534.6.6 by Robert Collins
Move find_repository to bzrdir, its not quite ideal there but its simpler and until someone chooses to vary the search by branch type its completely sufficient.
1395
        """Make a complete copy of the content in self into destination.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1396
1397
        This is a destructive operation! Do not use it on existing
1534.6.6 by Robert Collins
Move find_repository to bzrdir, its not quite ideal there but its simpler and until someone chooses to vary the search by branch type its completely sufficient.
1398
        repositories.
1399
        """
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1400
        return InterRepository.get(self, destination).copy_content(revision_id)
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1401
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1402
    def commit_write_group(self):
1403
        """Commit the contents accrued within the current write group.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1404
1405
        :seealso: start_write_group.
1406
        """
1407
        if self._write_group is not self.get_transaction():
1408
            # has an unlock or relock occured ?
2592.3.38 by Robert Collins
All experimental format tests passing again.
1409
            raise errors.BzrError('mismatched lock context %r and '
1410
                'write group %r.' %
1411
                (self.get_transaction(), self._write_group))
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1412
        self._commit_write_group()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1413
        self._write_group = None
1414
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1415
    def _commit_write_group(self):
1416
        """Template method for per-repository write group cleanup.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1417
1418
        This is called before the write group is considered to be
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1419
        finished and should ensure that all data handed to the repository
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1420
        for writing during the write group is safely committed (to the
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1421
        extent possible considering file system caching etc).
1422
        """
1423
4002.1.1 by Andrew Bennetts
Implement suspend_write_group/resume_write_group.
1424
    def suspend_write_group(self):
1425
        raise errors.UnsuspendableWriteGroup(self)
1426
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
1427
    def refresh_data(self):
1428
        """Re-read any data needed to to synchronise with disk.
1429
1430
        This method is intended to be called after another repository instance
1431
        (such as one used by a smart server) has inserted data into the
1432
        repository. It may not be called during a write group, but may be
1433
        called at any other time.
1434
        """
1435
        if self.is_in_write_group():
1436
            raise errors.InternalBzrError(
1437
                "May not refresh_data while in a write group.")
1438
        self._refresh_data()
1439
4002.1.1 by Andrew Bennetts
Implement suspend_write_group/resume_write_group.
1440
    def resume_write_group(self, tokens):
1441
        if not self.is_write_locked():
1442
            raise errors.NotWriteLocked(self)
1443
        if self._write_group:
1444
            raise errors.BzrError('already in a write group')
1445
        self._resume_write_group(tokens)
1446
        # so we can detect unlock/relock - the write group is now entered.
1447
        self._write_group = self.get_transaction()
4032.1.1 by John Arbash Meinel
Merge the removal of all trailing whitespace, and resolve conflicts.
1448
4002.1.1 by Andrew Bennetts
Implement suspend_write_group/resume_write_group.
1449
    def _resume_write_group(self, tokens):
1450
        raise errors.UnsuspendableWriteGroup(self)
1451
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1452
    def fetch(self, source, revision_id=None, pb=None, find_ghosts=False,
1453
            fetch_spec=None):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1454
        """Fetch the content required to construct revision_id from source.
1455
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1456
        If revision_id is None and fetch_spec is None, then all content is
1457
        copied.
1458
4145.1.1 by Robert Collins
Explicitly prevent fetching while the target repository is in a write group.
1459
        fetch() may not be used when the repository is in a write group -
1460
        either finish the current write group before using fetch, or use
1461
        fetch before starting the write group.
1462
2949.1.1 by Robert Collins
Change Repository.fetch to provide a find_ghosts parameter which triggers ghost filling.
1463
        :param find_ghosts: Find and copy revisions in the source that are
1464
            ghosts in the target (and not reachable directly by walking out to
1465
            the first-present revision in target from revision_id).
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1466
        :param revision_id: If specified, all the content needed for this
1467
            revision ID will be copied to the target.  Fetch will determine for
1468
            itself which content needs to be copied.
1469
        :param fetch_spec: If specified, a SearchResult or
1470
            PendingAncestryResult that describes which revisions to copy.  This
1471
            allows copying multiple heads at once.  Mutually exclusive with
1472
            revision_id.
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1473
        """
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1474
        if fetch_spec is not None and revision_id is not None:
1475
            raise AssertionError(
1476
                "fetch_spec and revision_id are mutually exclusive.")
4145.1.1 by Robert Collins
Explicitly prevent fetching while the target repository is in a write group.
1477
        if self.is_in_write_group():
4145.1.3 by Robert Collins
NEWS conflicts.
1478
            raise errors.InternalBzrError(
1479
                "May not fetch while in a write group.")
2592.3.115 by Robert Collins
Move same repository check up to Repository.fetch to allow all fetch implementations to benefit.
1480
        # fast path same-url fetch operations
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1481
        if self.has_same_location(source) and fetch_spec is None:
2592.3.115 by Robert Collins
Move same repository check up to Repository.fetch to allow all fetch implementations to benefit.
1482
            # check that last_revision is in 'from' and then return a
1483
            # no-operation.
1484
            if (revision_id is not None and
1485
                not _mod_revision.is_null(revision_id)):
1486
                self.get_revision(revision_id)
1487
            return 0, []
3582.1.3 by Martin Pool
Repository.fetch no longer needs to translate NotImplementedErro to IncompatibleRepositories
1488
        # if there is no specific appropriate InterRepository, this will get
1489
        # the InterRepository base class, which raises an
1490
        # IncompatibleRepositories when asked to fetch.
2323.8.3 by Aaron Bentley
Reduce scope of try/except, update NEWS
1491
        inter = InterRepository.get(source, self)
3582.1.3 by Martin Pool
Repository.fetch no longer needs to translate NotImplementedErro to IncompatibleRepositories
1492
        return inter.fetch(revision_id=revision_id, pb=pb,
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1493
            find_ghosts=find_ghosts, fetch_spec=fetch_spec)
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1494
2520.4.54 by Aaron Bentley
Hang a create_bundle method off repository
1495
    def create_bundle(self, target, base, fileobj, format=None):
1496
        return serializer.write_bundle(self, target, base, fileobj, format)
1497
2803.2.1 by Robert Collins
* CommitBuilder now advertises itself as requiring the root entry to be
1498
    def get_commit_builder(self, branch, parents, config, timestamp=None,
1499
                           timezone=None, committer=None, revprops=None,
1740.3.7 by Jelmer Vernooij
Move committer, log, revprops, timestamp and timezone to CommitBuilder.
1500
                           revision_id=None):
1501
        """Obtain a CommitBuilder for this repository.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1502
1740.3.7 by Jelmer Vernooij
Move committer, log, revprops, timestamp and timezone to CommitBuilder.
1503
        :param branch: Branch to commit to.
1504
        :param parents: Revision ids of the parents of the new revision.
1505
        :param config: Configuration to use.
1506
        :param timestamp: Optional timestamp recorded for commit.
1507
        :param timezone: Optional timezone for timestamp.
1508
        :param committer: Optional committer to set for commit.
1509
        :param revprops: Optional dictionary of revision properties.
1510
        :param revision_id: Optional revision id.
1511
        """
2818.3.2 by Robert Collins
Review feedback.
1512
        result = self._commit_builder_class(self, parents, config,
2592.3.135 by Robert Collins
Do not create many transient knit objects, saving 4% on commit.
1513
            timestamp, timezone, committer, revprops, revision_id)
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1514
        self.start_write_group()
1515
        return result
1740.3.1 by Jelmer Vernooij
Introduce and use CommitBuilder objects.
1516
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1517
    def unlock(self):
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1518
        if (self.control_files._lock_count == 1 and
1519
            self.control_files._lock_mode == 'w'):
1520
            if self._write_group is not None:
2592.3.244 by Martin Pool
unlock while in a write group now aborts the write group, unlocks, and errors.
1521
                self.abort_write_group()
1522
                self.control_files.unlock()
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1523
                raise errors.BzrError(
1524
                    'Must end write groups before releasing write locks.')
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1525
        self.control_files.unlock()
3882.6.23 by John Arbash Meinel
Change the XMLSerializer.read_inventory_from_string api.
1526
        if self.control_files._lock_count == 0:
1527
            self._inventory_entry_cache.clear()
3221.12.13 by Robert Collins
Implement generic stacking rather than pack-internals based stacking.
1528
        for repo in self._fallback_repositories:
1529
            repo.unlock()
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1530
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1531
    @needs_read_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
1532
    def clone(self, a_bzrdir, revision_id=None):
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1533
        """Clone this repository into a_bzrdir using the current format.
1534
1535
        Currently no check is made that the format of this repository and
1536
        the bzrdir format are compatible. FIXME RBC 20060201.
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1537
1538
        :return: The newly created destination repository.
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1539
        """
2440.1.1 by Martin Pool
Add new Repository.sprout,
1540
        # TODO: deprecate after 0.16; cloning this with all its settings is
1541
        # probably not very useful -- mbp 20070423
1542
        dest_repo = self._create_sprouting_repo(a_bzrdir, shared=self.is_shared())
1543
        self.copy_content_into(dest_repo, revision_id)
1544
        return dest_repo
1545
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1546
    def start_write_group(self):
1547
        """Start a write group in the repository.
1548
1549
        Write groups are used by repositories which do not have a 1:1 mapping
1550
        between file ids and backend store to manage the insertion of data from
1551
        both fetch and commit operations.
1552
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1553
        A write lock is required around the start_write_group/commit_write_group
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1554
        for the support of lock-requiring repository formats.
2617.6.8 by Robert Collins
Review feedback and documentation.
1555
1556
        One can only insert data into a repository inside a write group.
1557
2617.6.6 by Robert Collins
Some review feedback.
1558
        :return: None.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1559
        """
2592.3.188 by Robert Collins
Allow pack repositories to have multiple writers active at one time, for greater concurrency.
1560
        if not self.is_write_locked():
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1561
            raise errors.NotWriteLocked(self)
1562
        if self._write_group:
1563
            raise errors.BzrError('already in a write group')
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1564
        self._start_write_group()
1565
        # so we can detect unlock/relock - the write group is now entered.
2617.6.1 by Robert Collins
* New method on Repository - ``start_write_group``, ``end_write_group``
1566
        self._write_group = self.get_transaction()
1567
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1568
    def _start_write_group(self):
1569
        """Template method for per-repository write group startup.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1570
1571
        This is called before the write group is considered to be
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
1572
        entered.
1573
        """
1574
2440.1.1 by Martin Pool
Add new Repository.sprout,
1575
    @needs_read_lock
1576
    def sprout(self, to_bzrdir, revision_id=None):
1577
        """Create a descendent repository for new development.
1578
1579
        Unlike clone, this does not copy the settings of the repository.
1580
        """
1581
        dest_repo = self._create_sprouting_repo(to_bzrdir, shared=False)
1582
        dest_repo.fetch(self, revision_id=revision_id)
1583
        return dest_repo
1584
1585
    def _create_sprouting_repo(self, a_bzrdir, shared):
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1586
        if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):
1587
            # use target default format.
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1588
            dest_repo = a_bzrdir.create_repository()
1534.4.50 by Robert Collins
Got the bzrdir api straightened out, plenty of refactoring to use it pending, but the api is up and running.
1589
        else:
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1590
            # Most control formats need the repository to be specifically
1591
            # created, but on some old all-in-one formats it's not needed
1592
            try:
2440.1.1 by Martin Pool
Add new Repository.sprout,
1593
                dest_repo = self._format.initialize(a_bzrdir, shared=shared)
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
1594
            except errors.UninitializableFormat:
1595
                dest_repo = a_bzrdir.open_repository()
1596
        return dest_repo
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1597
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
1598
    def _get_sink(self):
1599
        """Return a sink for streaming into this repository."""
1600
        return StreamSink(self)
1601
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
1602
    def _get_source(self, to_format):
1603
        """Return a source for streaming from this repository."""
1604
        return StreamSource(self, to_format)
1605
1563.2.22 by Robert Collins
Move responsibility for repository.has_revision into RevisionStore
1606
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1607
    def has_revision(self, revision_id):
1563.2.22 by Robert Collins
Move responsibility for repository.has_revision into RevisionStore
1608
        """True if this repository has a copy of the revision."""
3172.3.1 by Robert Collins
Repository has a new method ``has_revisions`` which signals the presence
1609
        return revision_id in self.has_revisions((revision_id,))
1610
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1611
    @needs_read_lock
3172.3.1 by Robert Collins
Repository has a new method ``has_revisions`` which signals the presence
1612
    def has_revisions(self, revision_ids):
1613
        """Probe to find out the presence of multiple revisions.
1614
1615
        :param revision_ids: An iterable of revision_ids.
1616
        :return: A set of the revision_ids that were present.
1617
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1618
        parent_map = self.revisions.get_parent_map(
1619
            [(rev_id,) for rev_id in revision_ids])
1620
        result = set()
1621
        if _mod_revision.NULL_REVISION in revision_ids:
1622
            result.add(_mod_revision.NULL_REVISION)
1623
        result.update([key[0] for key in parent_map])
1624
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1625
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1626
    @needs_read_lock
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1627
    def get_revision(self, revision_id):
1628
        """Return the Revision object for a named revision."""
1629
        return self.get_revisions([revision_id])[0]
1630
1631
    @needs_read_lock
1570.1.13 by Robert Collins
Check for incorrect revision parentage in the weave during revision access.
1632
    def get_revision_reconcile(self, revision_id):
1633
        """'reconcile' helper routine that allows access to a revision always.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1634
1570.1.13 by Robert Collins
Check for incorrect revision parentage in the weave during revision access.
1635
        This variant of get_revision does not cross check the weave graph
1636
        against the revision one as get_revision does: but it should only
1637
        be used by reconcile, or reconcile-alike commands that are correcting
1638
        or testing the revision graph.
1639
        """
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1640
        return self._get_revisions([revision_id])[0]
2249.5.13 by John Arbash Meinel
Finish auditing Repository, and fix generate_ids to always generate utf8 ids.
1641
1756.1.2 by Aaron Bentley
Show logs using get_revisions
1642
    @needs_read_lock
1643
    def get_revisions(self, revision_ids):
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
1644
        """Get many revisions at once."""
1645
        return self._get_revisions(revision_ids)
1646
1647
    @needs_read_lock
1648
    def _get_revisions(self, revision_ids):
1649
        """Core work logic to get many revisions without sanity checks."""
1650
        for rev_id in revision_ids:
1651
            if not rev_id or not isinstance(rev_id, basestring):
1652
                raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1653
        keys = [(key,) for key in revision_ids]
1654
        stream = self.revisions.get_record_stream(keys, 'unordered', True)
1655
        revs = {}
1656
        for record in stream:
1657
            if record.storage_kind == 'absent':
1658
                raise errors.NoSuchRevision(self, record.key[0])
1659
            text = record.get_bytes_as('fulltext')
1660
            rev = self._serializer.read_revision_from_string(text)
1661
            revs[record.key[0]] = rev
1662
        return [revs[revid] for revid in revision_ids]
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1663
1185.65.27 by Robert Collins
Tweak storage towards mergability.
1664
    @needs_read_lock
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1665
    def get_revision_xml(self, revision_id):
4232.2.1 by Vincent Ladeuil
Stop-gap fix for Repository.get_revision_xml.
1666
        # TODO: jam 20070210 This shouldn't be necessary since get_revision
1667
        #       would have already do it.
1668
        # TODO: jam 20070210 Just use _serializer.write_revision_to_string()
1669
        # TODO: this can't just be replaced by:
1670
        # return self._serializer.write_revision_to_string(
1671
        #     self.get_revision(revision_id))
1672
        # as cStringIO preservers the encoding unlike write_revision_to_string
1673
        # or some other call down the path.
1674
        rev = self.get_revision(revision_id)
1675
        rev_tmp = cStringIO.StringIO()
1676
        # the current serializer..
1677
        self._serializer.write_revision(rev, rev_tmp)
1678
        rev_tmp.seek(0)
1679
        return rev_tmp.getvalue()
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1680
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1681
    def get_deltas_for_revisions(self, revisions, specific_fileids=None):
1756.3.19 by Aaron Bentley
Documentation and cleanups
1682
        """Produce a generator of revision deltas.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1683
1756.3.19 by Aaron Bentley
Documentation and cleanups
1684
        Note that the input is a sequence of REVISIONS, not revision_ids.
1685
        Trees will be held in memory until the generator exits.
1686
        Each delta is relative to the revision's lefthand predecessor.
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1687
1688
        :param specific_fileids: if not None, the result is filtered
1689
          so that only those file-ids, their parents and their
1690
          children are included.
1756.3.19 by Aaron Bentley
Documentation and cleanups
1691
        """
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1692
        # Get the revision-ids of interest
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1693
        required_trees = set()
1694
        for revision in revisions:
1695
            required_trees.add(revision.revision_id)
1696
            required_trees.update(revision.parent_ids[:1])
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1697
1698
        # Get the matching filtered trees. Note that it's more
1699
        # efficient to pass filtered trees to changes_from() rather
1700
        # than doing the filtering afterwards. changes_from() could
1701
        # arguably do the filtering itself but it's path-based, not
1702
        # file-id based, so filtering before or afterwards is
1703
        # currently easier.
1704
        if specific_fileids is None:
1705
            trees = dict((t.get_revision_id(), t) for
1706
                t in self.revision_trees(required_trees))
1707
        else:
1708
            trees = dict((t.get_revision_id(), t) for
1709
                t in self._filtered_revision_trees(required_trees,
1710
                specific_fileids))
1711
1712
        # Calculate the deltas
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1713
        for revision in revisions:
1714
            if not revision.parent_ids:
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
1715
                old_tree = self.revision_tree(_mod_revision.NULL_REVISION)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1716
            else:
1717
                old_tree = trees[revision.parent_ids[0]]
1852.10.3 by Robert Collins
Remove all uses of compare_trees and replace with Tree.changes_from throughout bzrlib.
1718
            yield trees[revision.revision_id].changes_from(old_tree)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1719
1756.3.19 by Aaron Bentley
Documentation and cleanups
1720
    @needs_read_lock
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1721
    def get_revision_delta(self, revision_id, specific_fileids=None):
1744.2.2 by Johan Rydberg
Add get_revision_delta to Repository; and make Branch.get_revision_delta use it.
1722
        """Return the delta for one revision.
1723
1724
        The delta is relative to the left-hand predecessor of the
1725
        revision.
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1726
1727
        :param specific_fileids: if not None, the result is filtered
1728
          so that only those file-ids, their parents and their
1729
          children are included.
1744.2.2 by Johan Rydberg
Add get_revision_delta to Repository; and make Branch.get_revision_delta use it.
1730
        """
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
1731
        r = self.get_revision(revision_id)
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
1732
        return list(self.get_deltas_for_revisions([r],
1733
            specific_fileids=specific_fileids))[0]
1744.2.2 by Johan Rydberg
Add get_revision_delta to Repository; and make Branch.get_revision_delta use it.
1734
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1735
    @needs_write_lock
1736
    def store_revision_signature(self, gpg_strategy, plaintext, revision_id):
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
1737
        signature = gpg_strategy.sign(plaintext)
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
1738
        self.add_signature_text(revision_id, signature)
2996.2.3 by Aaron Bentley
Add tests for install_revisions and add_signature
1739
1740
    @needs_write_lock
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
1741
    def add_signature_text(self, revision_id, signature):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1742
        self.signatures.add_lines((revision_id,), (),
1743
            osutils.split_lines(signature))
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
1744
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1745
    def find_text_key_references(self):
1746
        """Find the text key references within the repository.
1747
1748
        :return: A dictionary mapping text keys ((fileid, revision_id) tuples)
1749
            to whether they were referred to by the inventory of the
1750
            revision_id that they contain. The inventory texts from all present
1751
            revision ids are assessed to generate this report.
1752
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1753
        revision_keys = self.revisions.keys()
1754
        w = self.inventories
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1755
        pb = ui.ui_factory.nested_progress_bar()
1756
        try:
1757
            return self._find_text_key_references_from_xml_inventory_lines(
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1758
                w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
2988.1.2 by Robert Collins
New Repository API find_text_key_references for use by reconcile and check.
1759
        finally:
1760
            pb.finished()
1761
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1762
    def _find_text_key_references_from_xml_inventory_lines(self,
1763
        line_iterator):
1764
        """Core routine for extracting references to texts from inventories.
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1765
1766
        This performs the translation of xml lines to revision ids.
1767
2975.3.1 by Robert Collins
Change (without backwards compatibility) the
1768
        :param line_iterator: An iterator of lines, origin_version_id
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1769
        :return: A dictionary mapping text keys ((fileid, revision_id) tuples)
1770
            to whether they were referred to by the inventory of the
1771
            revision_id that they contain. Note that if that revision_id was
1772
            not part of the line_iterator's output then False will be given -
1773
            even though it may actually refer to that key.
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1774
        """
2988.2.2 by Robert Collins
Review feedback.
1775
        if not self._serializer.support_altered_by_hack:
1776
            raise AssertionError(
1777
                "_find_text_key_references_from_xml_inventory_lines only "
1778
                "supported for branches which store inventory as unnested xml"
1779
                ", not on %r" % self)
1694.2.6 by Martin Pool
[merge] bzr.dev
1780
        result = {}
1563.2.35 by Robert Collins
cleanup deprecation warnings and finish conversion so the inventory is knit based too.
1781
1694.2.6 by Martin Pool
[merge] bzr.dev
1782
        # this code needs to read every new line in every inventory for the
1783
        # inventories [revision_ids]. Seeing a line twice is ok. Seeing a line
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1784
        # not present in one of those inventories is unnecessary but not
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
1785
        # harmful because we are filtering by the revision id marker in the
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1786
        # inventory lines : we only select file ids altered in one of those
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
1787
        # revisions. We don't need to see all lines in the inventory because
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
1788
        # only those added in an inventory in rev X can contain a revision=X
1789
        # line.
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1790
        unescape_revid_cache = {}
1791
        unescape_fileid_cache = {}
1792
2163.2.5 by John Arbash Meinel
Inline the cache lookup, and explain why
1793
        # jam 20061218 In a big fetch, this handles hundreds of thousands
1794
        # of lines, so it has had a lot of inlining and optimizing done.
1795
        # Sorry that it is a little bit messy.
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1796
        # Move several functions to be local variables, since this is a long
1797
        # running loop.
1798
        search = self._file_ids_altered_regex.search
2163.2.5 by John Arbash Meinel
Inline the cache lookup, and explain why
1799
        unescape = _unescape_xml
2163.2.3 by John Arbash Meinel
Change to local variables to save another 300ms
1800
        setdefault = result.setdefault
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1801
        for line, line_key in line_iterator:
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1802
            match = search(line)
1803
            if match is None:
1804
                continue
1805
            # One call to match.group() returning multiple items is quite a
1806
            # bit faster than 2 calls to match.group() each returning 1
1807
            file_id, revision_id = match.group('file_id', 'revision_id')
1808
1809
            # Inlining the cache lookups helps a lot when you make 170,000
1810
            # lines and 350k ids, versus 8.4 unique ids.
1811
            # Using a cache helps in 2 ways:
1812
            #   1) Avoids unnecessary decoding calls
1813
            #   2) Re-uses cached strings, which helps in future set and
1814
            #      equality checks.
1815
            # (2) is enough that removing encoding entirely along with
1816
            # the cache (so we are using plain strings) results in no
1817
            # performance improvement.
1818
            try:
1819
                revision_id = unescape_revid_cache[revision_id]
1820
            except KeyError:
1821
                unescaped = unescape(revision_id)
1822
                unescape_revid_cache[revision_id] = unescaped
1823
                revision_id = unescaped
1824
2988.2.2 by Robert Collins
Review feedback.
1825
            # Note that unconditionally unescaping means that we deserialise
1826
            # every fileid, which for general 'pull' is not great, but we don't
1827
            # really want to have some many fulltexts that this matters anyway.
1828
            # RBC 20071114.
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1829
            try:
1830
                file_id = unescape_fileid_cache[file_id]
1831
            except KeyError:
1832
                unescaped = unescape(file_id)
1833
                unescape_fileid_cache[file_id] = unescaped
1834
                file_id = unescaped
1835
1836
            key = (file_id, revision_id)
1837
            setdefault(key, False)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1838
            if revision_id == line_key[-1]:
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1839
                result[key] = True
1840
        return result
1841
3735.2.135 by Robert Collins
Permit fetching bzr.dev [deal with inconsistent inventories.]
1842
    def _inventory_xml_lines_for_keys(self, keys):
1843
        """Get a line iterator of the sort needed for findind references.
1844
1845
        Not relevant for non-xml inventory repositories.
1846
1847
        Ghosts in revision_keys are ignored.
1848
1849
        :param revision_keys: The revision keys for the inventories to inspect.
1850
        :return: An iterator over (inventory line, revid) for the fulltexts of
1851
            all of the xml inventories specified by revision_keys.
1852
        """
1853
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
1854
        for record in stream:
1855
            if record.storage_kind != 'absent':
1856
                chunks = record.get_bytes_as('chunked')
3735.2.136 by Robert Collins
Fix typo
1857
                revid = record.key[-1]
3735.2.135 by Robert Collins
Permit fetching bzr.dev [deal with inconsistent inventories.]
1858
                lines = osutils.chunks_to_lines(chunks)
1859
                for line in lines:
1860
                    yield line, revid
1861
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1862
    def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
1863
        revision_ids):
1864
        """Helper routine for fileids_altered_by_revision_ids.
1865
1866
        This performs the translation of xml lines to revision ids.
1867
1868
        :param line_iterator: An iterator of lines, origin_version_id
1869
        :param revision_ids: The revision ids to filter for. This should be a
1870
            set or other type which supports efficient __contains__ lookups, as
1871
            the revision id from each parsed line will be looked up in the
1872
            revision_ids filter.
1873
        :return: a dictionary mapping altered file-ids to an iterable of
1874
        revision_ids. Each altered file-ids has the exact revision_ids that
1875
        altered it listed explicitly.
1876
        """
3735.2.135 by Robert Collins
Permit fetching bzr.dev [deal with inconsistent inventories.]
1877
        seen = set(self._find_text_key_references_from_xml_inventory_lines(
1878
                line_iterator).iterkeys())
1879
        # Note that revision_ids are revision keys.
1880
        parent_maps = self.revisions.get_parent_map(revision_ids)
1881
        parents = set()
1882
        map(parents.update, parent_maps.itervalues())
1883
        parents.difference_update(revision_ids)
1884
        parent_seen = set(self._find_text_key_references_from_xml_inventory_lines(
1885
            self._inventory_xml_lines_for_keys(parents)))
1886
        new_keys = seen - parent_seen
2988.1.1 by Robert Collins
Refactor fetch's xml inventory parsing into a core routine that extracts the data and a separate one that filters for fetch.
1887
        result = {}
1888
        setdefault = result.setdefault
3735.2.135 by Robert Collins
Permit fetching bzr.dev [deal with inconsistent inventories.]
1889
        for key in new_keys:
1890
            setdefault(key[0], set()).add(key[-1])
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1891
        return result
1892
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
1893
    def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1894
        """Find the file ids and versions affected by revisions.
1895
1896
        :param revisions: an iterable containing revision ids.
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
1897
        :param _inv_weave: The inventory weave from this repository or None.
1898
            If None, the inventory weave will be opened automatically.
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1899
        :return: a dictionary mapping altered file-ids to an iterable of
1900
        revision_ids. Each altered file-ids has the exact revision_ids that
1901
        altered it listed explicitly.
1902
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1903
        selected_keys = set((revid,) for revid in revision_ids)
1904
        w = _inv_weave or self.inventories
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
1905
        pb = ui.ui_factory.nested_progress_bar()
1906
        try:
2592.3.110 by Robert Collins
Filter out texts and signatures not referenced by the revisions being copied during pack to pack fetching.
1907
            return self._find_file_ids_from_xml_inventory_lines(
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1908
                w.iter_lines_added_or_present_in_keys(
1909
                    selected_keys, pb=pb),
1910
                selected_keys)
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
1911
        finally:
1912
            pb.finished()
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
1913
2708.1.7 by Aaron Bentley
Rename extract_files_bytes to iter_files_bytes
1914
    def iter_files_bytes(self, desired_files):
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1915
        """Iterate through file versions.
1916
2708.1.10 by Aaron Bentley
Update docstrings
1917
        Files will not necessarily be returned in the order they occur in
1918
        desired_files.  No specific order is guaranteed.
1919
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1920
        Yields pairs of identifier, bytes_iterator.  identifier is an opaque
2708.1.10 by Aaron Bentley
Update docstrings
1921
        value supplied by the caller as part of desired_files.  It should
1922
        uniquely identify the file version in the caller's context.  (Examples:
1923
        an index number or a TreeTransform trans_id.)
1924
1925
        bytes_iterator is an iterable of bytestrings for the file.  The
1926
        kind of iterable and length of the bytestrings are unspecified, but for
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1927
        this implementation, it is a list of bytes produced by
1928
        VersionedFile.get_record_stream().
2708.1.10 by Aaron Bentley
Update docstrings
1929
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1930
        :param desired_files: a list of (file_id, revision_id, identifier)
2708.1.10 by Aaron Bentley
Update docstrings
1931
            triples
2708.1.9 by Aaron Bentley
Clean-up docs and imports
1932
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1933
        text_keys = {}
2708.1.3 by Aaron Bentley
Implement extract_files_bytes on Repository
1934
        for file_id, revision_id, callable_data in desired_files:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1935
            text_keys[(file_id, revision_id)] = callable_data
1936
        for record in self.texts.get_record_stream(text_keys, 'unordered', True):
1937
            if record.storage_kind == 'absent':
1938
                raise errors.RevisionNotPresent(record.key, self)
4202.1.1 by John Arbash Meinel
Update Repository.iter_files_bytes() to return an iterable of bytestrings.
1939
            yield text_keys[record.key], record.get_bytes_as('chunked')
2708.1.3 by Aaron Bentley
Implement extract_files_bytes on Repository
1940
3063.2.1 by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted.
1941
    def _generate_text_key_index(self, text_key_references=None,
1942
        ancestors=None):
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1943
        """Generate a new text key index for the repository.
1944
1945
        This is an expensive function that will take considerable time to run.
1946
1947
        :return: A dict mapping text keys ((file_id, revision_id) tuples) to a
1948
            list of parents, also text keys. When a given key has no parents,
1949
            the parents list will be [NULL_REVISION].
1950
        """
1951
        # All revisions, to find inventory parents.
3063.2.1 by Robert Collins
Solve reconciling erroring when multiple portions of a single delta chain are being reinserted.
1952
        if ancestors is None:
3287.6.1 by Robert Collins
* ``VersionedFile.get_graph`` is deprecated, with no replacement method.
1953
            graph = self.get_graph()
1954
            ancestors = graph.get_parent_map(self.all_revision_ids())
2951.2.9 by Robert Collins
* ``pack-0.92`` repositories can now be reconciled.
1955
        if text_key_references is None:
1956
            text_key_references = self.find_text_key_references()
2988.3.1 by Robert Collins
Handle the progress bar in _generate_text_key_index correctly.
1957
        pb = ui.ui_factory.nested_progress_bar()
1958
        try:
1959
            return self._do_generate_text_key_index(ancestors,
1960
                text_key_references, pb)
1961
        finally:
1962
            pb.finished()
1963
1964
    def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
1965
        """Helper for _generate_text_key_index to avoid deep nesting."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1966
        revision_order = tsort.topo_sort(ancestors)
1967
        invalid_keys = set()
1968
        revision_keys = {}
1969
        for revision_id in revision_order:
1970
            revision_keys[revision_id] = set()
1971
        text_count = len(text_key_references)
1972
        # a cache of the text keys to allow reuse; costs a dict of all the
1973
        # keys, but saves a 2-tuple for every child of a given key.
1974
        text_key_cache = {}
1975
        for text_key, valid in text_key_references.iteritems():
1976
            if not valid:
1977
                invalid_keys.add(text_key)
1978
            else:
1979
                revision_keys[text_key[1]].add(text_key)
1980
            text_key_cache[text_key] = text_key
1981
        del text_key_references
1982
        text_index = {}
1983
        text_graph = graph.Graph(graph.DictParentsProvider(text_index))
1984
        NULL_REVISION = _mod_revision.NULL_REVISION
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
1985
        # Set a cache with a size of 10 - this suffices for bzr.dev but may be
1986
        # too small for large or very branchy trees. However, for 55K path
1987
        # trees, it would be easy to use too much memory trivially. Ideally we
1988
        # could gauge this by looking at available real memory etc, but this is
1989
        # always a tricky proposition.
1990
        inventory_cache = lru_cache.LRUCache(10)
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1991
        batch_size = 10 # should be ~150MB on a 55K path tree
1992
        batch_count = len(revision_order) / batch_size + 1
1993
        processed_texts = 0
4103.3.2 by Martin Pool
Remove trailing punctuation from progress messages
1994
        pb.update("Calculating text parents", processed_texts, text_count)
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
1995
        for offset in xrange(batch_count):
1996
            to_query = revision_order[offset * batch_size:(offset + 1) *
1997
                batch_size]
1998
            if not to_query:
1999
                break
2000
            for rev_tree in self.revision_trees(to_query):
2001
                revision_id = rev_tree.get_revision_id()
2002
                parent_ids = ancestors[revision_id]
2003
                for text_key in revision_keys[revision_id]:
4103.3.2 by Martin Pool
Remove trailing punctuation from progress messages
2004
                    pb.update("Calculating text parents", processed_texts)
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
2005
                    processed_texts += 1
2006
                    candidate_parents = []
2007
                    for parent_id in parent_ids:
2008
                        parent_text_key = (text_key[0], parent_id)
2009
                        try:
2010
                            check_parent = parent_text_key not in \
2011
                                revision_keys[parent_id]
2012
                        except KeyError:
2013
                            # the parent parent_id is a ghost:
2014
                            check_parent = False
2015
                            # truncate the derived graph against this ghost.
2016
                            parent_text_key = None
2017
                        if check_parent:
2018
                            # look at the parent commit details inventories to
2019
                            # determine possible candidates in the per file graph.
2020
                            # TODO: cache here.
2988.1.5 by Robert Collins
Use a LRU cache when generating the text index to reduce inventory deserialisations.
2021
                            try:
2022
                                inv = inventory_cache[parent_id]
2023
                            except KeyError:
2024
                                inv = self.revision_tree(parent_id).inventory
2025
                                inventory_cache[parent_id] = inv
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
2026
                            try:
2027
                                parent_entry = inv[text_key[0]]
2028
                            except (KeyError, errors.NoSuchId):
2029
                                parent_entry = None
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
2030
                            if parent_entry is not None:
2031
                                parent_text_key = (
2032
                                    text_key[0], parent_entry.revision)
2033
                            else:
2034
                                parent_text_key = None
2035
                        if parent_text_key is not None:
2036
                            candidate_parents.append(
2037
                                text_key_cache[parent_text_key])
2038
                    parent_heads = text_graph.heads(candidate_parents)
2039
                    new_parents = list(parent_heads)
2040
                    new_parents.sort(key=lambda x:candidate_parents.index(x))
2041
                    if new_parents == []:
2042
                        new_parents = [NULL_REVISION]
2043
                    text_index[text_key] = new_parents
2044
2045
        for text_key in invalid_keys:
2046
            text_index[text_key] = [NULL_REVISION]
2047
        return text_index
2048
2668.2.8 by Andrew Bennetts
Rename get_data_to_fetch_for_revision_ids as item_keys_introduced_by.
2049
    def item_keys_introduced_by(self, revision_ids, _files_pb=None):
2050
        """Get an iterable listing the keys of all the data introduced by a set
2051
        of revision IDs.
2052
2053
        The keys will be ordered so that the corresponding items can be safely
2054
        fetched and inserted in that order.
2055
2056
        :returns: An iterable producing tuples of (knit-kind, file-id,
2057
            versions).  knit-kind is one of 'file', 'inventory', 'signatures',
2058
            'revisions'.  file-id is None unless knit-kind is 'file'.
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2059
        """
3735.4.4 by Andrew Bennetts
Change the layering, to put the custom file_id list underneath item_keys_intoduced_by
2060
        for result in self._find_file_keys_to_fetch(revision_ids, _files_pb):
2061
            yield result
2062
        del _files_pb
2063
        for result in self._find_non_file_keys_to_fetch(revision_ids):
2064
            yield result
2065
2066
    def _find_file_keys_to_fetch(self, revision_ids, pb):
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2067
        # XXX: it's a bit weird to control the inventory weave caching in this
2535.3.7 by Andrew Bennetts
Remove now unused _fetch_weave_texts, make progress reporting closer to how it was before I refactored __fetch.
2068
        # generator.  Ideally the caching would be done in fetch.py I think.  Or
2069
        # maybe this generator should explicitly have the contract that it
2070
        # should not be iterated until the previously yielded item has been
2071
        # processed?
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2072
        inv_w = self.inventories
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2073
2074
        # file ids that changed
3422.1.1 by John Arbash Meinel
merge in bzr-1.5rc1, revert the transaction cache change
2075
        file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
2535.3.8 by Andrew Bennetts
Unbreak progress reporting.
2076
        count = 0
2077
        num_file_ids = len(file_ids)
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2078
        for file_id, altered_versions in file_ids.iteritems():
3735.4.4 by Andrew Bennetts
Change the layering, to put the custom file_id list underneath item_keys_intoduced_by
2079
            if pb is not None:
2080
                pb.update("fetch texts", count, num_file_ids)
2535.3.8 by Andrew Bennetts
Unbreak progress reporting.
2081
            count += 1
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2082
            yield ("file", file_id, altered_versions)
2083
3735.4.4 by Andrew Bennetts
Change the layering, to put the custom file_id list underneath item_keys_intoduced_by
2084
    def _find_non_file_keys_to_fetch(self, revision_ids):
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2085
        # inventory
2086
        yield ("inventory", None, revision_ids)
2087
2088
        # signatures
3825.5.2 by Andrew Bennetts
Ensure that item_keys_introduced_by returns the
2089
        # XXX: Note ATM no callers actually pay attention to this return
2090
        #      instead they just use the list of revision ids and ignore
2091
        #      missing sigs. Consider removing this work entirely
2092
        revisions_with_signatures = set(self.signatures.get_parent_map(
2093
            [(r,) for r in revision_ids]))
3825.5.1 by Andrew Bennetts
Improve determining signatures to transfer in item_keys_introduced_by.
2094
        revisions_with_signatures = set(
3825.5.2 by Andrew Bennetts
Ensure that item_keys_introduced_by returns the
2095
            [r for (r,) in revisions_with_signatures])
3825.5.1 by Andrew Bennetts
Improve determining signatures to transfer in item_keys_introduced_by.
2096
        revisions_with_signatures.intersection_update(revision_ids)
2535.3.25 by Andrew Bennetts
Fetch signatures too.
2097
        yield ("signatures", None, revisions_with_signatures)
2535.3.6 by Andrew Bennetts
Move some "what repo data to fetch logic" from RepoFetcher to Repository.
2098
2099
        # revisions
2100
        yield ("revisions", None, revision_ids)
2101
1185.65.27 by Robert Collins
Tweak storage towards mergability.
2102
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2103
    def get_inventory(self, revision_id):
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
2104
        """Get Inventory object by revision id."""
2105
        return self.iter_inventories([revision_id]).next()
2106
2107
    def iter_inventories(self, revision_ids):
2108
        """Get many inventories by revision_ids.
2109
2110
        This will buffer some or all of the texts used in constructing the
2111
        inventories in memory, but will only parse a single inventory at a
2112
        time.
2113
4202.2.1 by Ian Clatworthy
get directory logging working again
2114
        :param revision_ids: The expected revision ids of the inventories.
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
2115
        :return: An iterator of inventories.
2116
        """
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
2117
        if ((None in revision_ids)
2118
            or (_mod_revision.NULL_REVISION in revision_ids)):
2119
            raise ValueError('cannot get null revision inventory')
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
2120
        return self._iter_inventories(revision_ids)
2121
2122
    def _iter_inventories(self, revision_ids):
2123
        """single-document based inventory iteration."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2124
        for text, revision_id in self._iter_inventory_xmls(revision_ids):
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
2125
            yield self.deserialise_inventory(revision_id, text)
1740.2.3 by Aaron Bentley
Only reserialize the working tree basis inventory when needed.
2126
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2127
    def _iter_inventory_xmls(self, revision_ids):
2128
        keys = [(revision_id,) for revision_id in revision_ids]
2129
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
3890.2.3 by John Arbash Meinel
Use the 'chunked' interface to keep memory consumption minimal during revision_trees()
2130
        text_chunks = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2131
        for record in stream:
2132
            if record.storage_kind != 'absent':
3890.2.3 by John Arbash Meinel
Use the 'chunked' interface to keep memory consumption minimal during revision_trees()
2133
                text_chunks[record.key] = record.get_bytes_as('chunked')
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2134
            else:
2135
                raise errors.NoSuchRevision(self, record.key)
2136
        for key in keys:
3890.2.3 by John Arbash Meinel
Use the 'chunked' interface to keep memory consumption minimal during revision_trees()
2137
            chunks = text_chunks.pop(key)
2138
            yield ''.join(chunks), key[-1]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2139
1740.2.3 by Aaron Bentley
Only reserialize the working tree basis inventory when needed.
2140
    def deserialise_inventory(self, revision_id, xml):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2141
        """Transform the xml into an inventory object.
1740.2.3 by Aaron Bentley
Only reserialize the working tree basis inventory when needed.
2142
2143
        :param revision_id: The expected revision id of the inventory.
2144
        :param xml: A serialised inventory.
2145
        """
3882.6.23 by John Arbash Meinel
Change the XMLSerializer.read_inventory_from_string api.
2146
        result = self._serializer.read_inventory_from_string(xml, revision_id,
2147
                    entry_cache=self._inventory_entry_cache)
3169.2.3 by Robert Collins
Use an if, not an assert, as we test with -O.
2148
        if result.revision_id != revision_id:
2149
            raise AssertionError('revision id mismatch %s != %s' % (
2150
                result.revision_id, revision_id))
3169.2.2 by Robert Collins
Add a test to Repository.deserialise_inventory that the resulting ivnentory is the one asked for, and update relevant tests. Also tweak the model 1 to 2 regenerate inventories logic to use the revision trees parent marker which is more accurate in some cases.
2151
        return result
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2152
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
2153
    def serialise_inventory(self, inv):
1910.2.48 by Aaron Bentley
Update from review comments
2154
        return self._serializer.write_inventory_to_string(inv)
1910.2.22 by Aaron Bentley
Make commits preserve root entry data
2155
2817.2.1 by Robert Collins
* Inventory serialisation no longer double-sha's the content.
2156
    def _serialise_inventory_to_lines(self, inv):
2157
        return self._serializer.write_inventory_to_lines(inv)
2158
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
2159
    def get_serializer_format(self):
2160
        return self._serializer.format_num
2161
1185.65.27 by Robert Collins
Tweak storage towards mergability.
2162
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2163
    def get_inventory_xml(self, revision_id):
2164
        """Get inventory XML as a file object."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2165
        texts = self._iter_inventory_xmls([revision_id])
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2166
        try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2167
            text, revision_id = texts.next()
2168
        except StopIteration:
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
2169
            raise errors.HistoryMissing(self, 'inventory', revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2170
        return text
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2171
1185.65.27 by Robert Collins
Tweak storage towards mergability.
2172
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2173
    def get_inventory_sha1(self, revision_id):
2174
        """Return the sha1 hash of the inventory entry
2175
        """
2176
        return self.get_revision(revision_id).inventory_sha1
2177
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
2178
    def iter_reverse_revision_history(self, revision_id):
2179
        """Iterate backwards through revision ids in the lefthand history
2180
2181
        :param revision_id: The revision id to start with.  All its lefthand
2182
            ancestors will be traversed.
2183
        """
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
2184
        graph = self.get_graph()
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
2185
        next_id = revision_id
2186
        while True:
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
2187
            if next_id in (None, _mod_revision.NULL_REVISION):
2188
                return
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
2189
            yield next_id
3287.5.10 by Robert Collins
Note iter_reverse_revision_history exception decision.
2190
            # Note: The following line may raise KeyError in the event of
2191
            # truncated history. We decided not to have a try:except:raise
2192
            # RevisionNotPresent here until we see a use for it, because of the
2193
            # cost in an inner loop that is by its very nature O(history).
2194
            # Robert Collins 20080326
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
2195
            parents = graph.get_parent_map([next_id])[next_id]
2230.3.54 by Aaron Bentley
Move reverse history iteration to repository
2196
            if len(parents) == 0:
2197
                return
2198
            else:
2199
                next_id = parents[0]
2200
1594.2.3 by Robert Collins
bugfix revision.MultipleRevisionSources.get_revision_graph to integrate ghosts between sources. [slow on weaves, fast on knits.
2201
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2202
    def get_revision_inventory(self, revision_id):
2203
        """Return inventory of a past revision."""
2204
        # TODO: Unify this with get_inventory()
2205
        # bzr 0.0.6 and later imposes the constraint that the inventory_id
2206
        # must be the same as its revision, so this is trivial.
1534.4.28 by Robert Collins
first cut at merge from integration.
2207
        if revision_id is None:
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2208
            # This does not make sense: if there is no revision,
2209
            # then it is the current tree inventory surely ?!
2210
            # and thus get_root_id() is something that looks at the last
2211
            # commit on the branch, and the get_root_id is an inventory check.
2212
            raise NotImplementedError
2213
            # return Inventory(self.get_root_id())
2214
        else:
2215
            return self.get_inventory(revision_id)
2216
1534.6.3 by Robert Collins
find_repository sufficiently robust.
2217
    def is_shared(self):
2218
        """Return True if this repository is flagged as a shared repository."""
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2219
        raise NotImplementedError(self.is_shared)
1534.6.3 by Robert Collins
find_repository sufficiently robust.
2220
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
2221
    @needs_write_lock
1692.1.1 by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow
2222
    def reconcile(self, other=None, thorough=False):
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
2223
        """Reconcile this repository."""
2224
        from bzrlib.reconcile import RepoReconciler
1692.1.1 by Robert Collins
* Repository.reconcile now takes a thorough keyword parameter to allow
2225
        reconciler = RepoReconciler(self, thorough=thorough)
1594.2.7 by Robert Collins
Add versionedfile.fix_parents api for correcting data post hoc.
2226
        reconciler.reconcile()
2227
        return reconciler
2440.1.1 by Martin Pool
Add new Repository.sprout,
2228
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
2229
    def _refresh_data(self):
2230
        """Helper called from lock_* to ensure coherency with disk.
2231
2232
        The default implementation does nothing; it is however possible
2233
        for repositories to maintain loaded indices across multiple locks
2234
        by checking inside their implementation of this method to see
2235
        whether their indices are still valid. This depends of course on
4145.1.2 by Robert Collins
Add a refresh_data method on Repository allowing cleaner handling of insertions into RemoteRepository objects with _real_repository instances.
2236
        the disk format being validatable in this manner. This method is
2237
        also called by the refresh_data() public interface to cause a refresh
2238
        to occur while in a write lock so that data inserted by a smart server
2239
        push operation is visible on the client's instance of the physical
2240
        repository.
2617.6.2 by Robert Collins
Add abort_write_group and wire write_groups into fetch and commit.
2241
        """
2242
1534.6.3 by Robert Collins
find_repository sufficiently robust.
2243
    @needs_read_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2244
    def revision_tree(self, revision_id):
2245
        """Return Tree for a revision on this branch.
2246
3668.5.2 by Jelmer Vernooij
Fix docstring.
2247
        `revision_id` may be NULL_REVISION for the empty tree revision.
1852.5.1 by Robert Collins
Deprecate EmptyTree in favour of using Repository.revision_tree.
2248
        """
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
2249
        revision_id = _mod_revision.ensure_null(revision_id)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2250
        # TODO: refactor this to use an existing revision object
2251
        # so we don't need to read it in twice.
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
2252
        if revision_id == _mod_revision.NULL_REVISION:
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2253
            return RevisionTree(self, Inventory(root_id=None),
1731.1.61 by Aaron Bentley
Merge bzr.dev
2254
                                _mod_revision.NULL_REVISION)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2255
        else:
2256
            inv = self.get_revision_inventory(revision_id)
1185.65.17 by Robert Collins
Merge from integration, mode-changes are broken.
2257
            return RevisionTree(self, inv, revision_id)
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2258
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
2259
    def revision_trees(self, revision_ids):
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
2260
        """Return Trees for revisions in this repository.
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
2261
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
2262
        :param revision_ids: a sequence of revision-ids;
2263
          a revision-id may not be None or 'null:'
2264
        """
3169.2.1 by Robert Collins
New method ``iter_inventories`` on Repository for access to many
2265
        inventories = self.iter_inventories(revision_ids)
2266
        for inv in inventories:
2267
            yield RevisionTree(self, inv, inv.revision_id)
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
2268
4137.3.2 by Ian Clatworthy
Repository.get_deltas_for_revisions() now supports file-id filtering
2269
    def _filtered_revision_trees(self, revision_ids, file_ids):
2270
        """Return Tree for a revision on this branch with only some files.
2271
2272
        :param revision_ids: a sequence of revision-ids;
2273
          a revision-id may not be None or 'null:'
2274
        :param file_ids: if not None, the result is filtered
2275
          so that only those file-ids, their parents and their
2276
          children are included.
2277
        """
2278
        inventories = self.iter_inventories(revision_ids)
2279
        for inv in inventories:
2280
            # Should we introduce a FilteredRevisionTree class rather
2281
            # than pre-filter the inventory here?
2282
            filtered_inv = inv.filter(file_ids)
2283
            yield RevisionTree(self, filtered_inv, filtered_inv.revision_id)
2284
1756.3.3 by Aaron Bentley
More refactoring, introduce revision_trees.
2285
    @needs_read_lock
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
2286
    def get_ancestry(self, revision_id, topo_sorted=True):
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
2287
        """Return a list of revision-ids integrated by a revision.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
2288
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2289
        The first element of the list is always None, indicating the origin
2290
        revision.  This might change when we have history horizons, or
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
2291
        perhaps we should have a new API.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2292
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
2293
        This is topologically sorted.
2294
        """
2598.5.1 by Aaron Bentley
Start eliminating the use of None to indicate null revision
2295
        if _mod_revision.is_null(revision_id):
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
2296
            return [None]
1534.4.41 by Robert Collins
Branch now uses BzrDir reasonably sanely.
2297
        if not self.has_revision(revision_id):
2298
            raise errors.NoSuchRevision(self, revision_id)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2299
        graph = self.get_graph()
2300
        keys = set()
2301
        search = graph._make_breadth_first_searcher([revision_id])
2302
        while True:
2303
            try:
2304
                found, ghosts = search.next_with_ghosts()
2305
            except StopIteration:
2306
                break
2307
            keys.update(found)
2308
        if _mod_revision.NULL_REVISION in keys:
2309
            keys.remove(_mod_revision.NULL_REVISION)
2310
        if topo_sorted:
2311
            parent_map = graph.get_parent_map(keys)
2312
            keys = tsort.topo_sort(parent_map)
2313
        return [None] + list(keys)
1185.66.2 by Aaron Bentley
Moved get_ancestry to RevisionStorage
2314
2604.2.1 by Robert Collins
(robertc) Introduce a pack command.
2315
    def pack(self):
2316
        """Compress the data within the repository.
2317
2318
        This operation only makes sense for some repository types. For other
2319
        types it should be a no-op that just returns.
2320
2321
        This stub method does not require a lock, but subclasses should use
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2322
        @needs_write_lock as this is a long running call its reasonable to
2604.2.1 by Robert Collins
(robertc) Introduce a pack command.
2323
        implicitly lock for the user.
2324
        """
2325
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2326
    def get_transaction(self):
2327
        return self.control_files.get_transaction()
2328
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
2329
    def get_parent_map(self, revision_ids):
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
2330
        """See graph._StackedParentsProvider.get_parent_map"""
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
2331
        # revisions index works in keys; this just works in revisions
2332
        # therefore wrap and unwrap
2333
        query_keys = []
2334
        result = {}
2335
        for revision_id in revision_ids:
2336
            if revision_id == _mod_revision.NULL_REVISION:
2337
                result[revision_id] = ()
2338
            elif revision_id is None:
3373.5.2 by John Arbash Meinel
Add repository_implementation tests for get_parent_map
2339
                raise ValueError('get_parent_map(None) is not valid')
3517.4.17 by Martin Pool
Redo base Repository.get_parent_map to use .revisions graph
2340
            else:
2341
                query_keys.append((revision_id ,))
2342
        for ((revision_id,), parent_keys) in \
2343
                self.revisions.get_parent_map(query_keys).iteritems():
2344
            if parent_keys:
2345
                result[revision_id] = tuple(parent_revid
2346
                    for (parent_revid,) in parent_keys)
2347
            else:
2348
                result[revision_id] = (_mod_revision.NULL_REVISION,)
2349
        return result
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
2350
2351
    def _make_parents_provider(self):
2352
        return self
2353
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
2354
    def get_graph(self, other_repository=None):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
2355
        """Return the graph walker for this repository format"""
2356
        parents_provider = self._make_parents_provider()
2490.2.14 by Aaron Bentley
Avoid StackedParentsProvider when underlying repos match
2357
        if (other_repository is not None and
3211.3.1 by Jelmer Vernooij
Use convenience function to check whether two repository handles are referring to the same repository.
2358
            not self.has_same_location(other_repository)):
2490.2.21 by Aaron Bentley
Rename graph to deprecated_graph
2359
            parents_provider = graph._StackedParentsProvider(
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
2360
                [parents_provider, other_repository._make_parents_provider()])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
2361
        return graph.Graph(parents_provider)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
2362
4145.2.1 by Ian Clatworthy
faster check
2363
    def _get_versioned_file_checker(self, text_key_references=None):
2364
        """Return an object suitable for checking versioned files.
2365
        
2366
        :param text_key_references: if non-None, an already built
2367
            dictionary mapping text keys ((fileid, revision_id) tuples)
2368
            to whether they were referred to by the inventory of the
2369
            revision_id that they contain. If None, this will be
2370
            calculated.
2371
        """
2372
        return _VersionedFileChecker(self,
2373
            text_key_references=text_key_references)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
2374
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2375
    def revision_ids_to_search_result(self, result_set):
2376
        """Convert a set of revision ids to a graph SearchResult."""
2377
        result_parents = set()
2378
        for parents in self.get_graph().get_parent_map(
2379
            result_set).itervalues():
2380
            result_parents.update(parents)
2381
        included_keys = result_set.intersection(result_parents)
2382
        start_keys = result_set.difference(included_keys)
2383
        exclude_keys = result_parents.difference(result_set)
2384
        result = graph.SearchResult(start_keys, exclude_keys,
2385
            len(result_set), result_set)
2386
        return result
2387
1185.65.27 by Robert Collins
Tweak storage towards mergability.
2388
    @needs_write_lock
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
2389
    def set_make_working_trees(self, new_value):
2390
        """Set the policy flag for making working trees when creating branches.
2391
2392
        This only applies to branches that use this repository.
2393
2394
        The default is 'True'.
2395
        :param new_value: True to restore the default, False to disable making
2396
                          working trees.
2397
        """
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2398
        raise NotImplementedError(self.set_make_working_trees)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2399
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
2400
    def make_working_trees(self):
2401
        """Returns the policy for making working trees on new branches."""
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2402
        raise NotImplementedError(self.make_working_trees)
1534.6.5 by Robert Collins
Cloning of repos preserves shared and make-working-tree attributes.
2403
2404
    @needs_write_lock
1185.65.1 by Aaron Bentley
Refactored out ControlFiles and RevisionStore from _Branch
2405
    def sign_revision(self, revision_id, gpg_strategy):
2406
        plaintext = Testament.from_revision(self, revision_id).as_short_text()
2407
        self.store_revision_signature(gpg_strategy, plaintext, revision_id)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2408
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
2409
    @needs_read_lock
2410
    def has_signature_for_revision_id(self, revision_id):
2411
        """Query for a revision signature for revision_id in the repository."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2412
        if not self.has_revision(revision_id):
2413
            raise errors.NoSuchRevision(self, revision_id)
2414
        sig_present = (1 == len(
2415
            self.signatures.get_parent_map([(revision_id,)])))
2416
        return sig_present
1563.2.29 by Robert Collins
Remove all but fetch references to repository.revision_store.
2417
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2418
    @needs_read_lock
2419
    def get_signature_text(self, revision_id):
2420
        """Return the text for a signature."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2421
        stream = self.signatures.get_record_stream([(revision_id,)],
2422
            'unordered', True)
2423
        record = stream.next()
2424
        if record.storage_kind == 'absent':
2425
            raise errors.NoSuchRevision(self, revision_id)
2426
        return record.get_bytes_as('fulltext')
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
2427
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
2428
    @needs_read_lock
2745.6.36 by Andrew Bennetts
Deprecate revision_ids arg to Repository.check and other tweaks.
2429
    def check(self, revision_ids=None):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
2430
        """Check consistency of all history of given revision_ids.
2431
2432
        Different repository implementations should override _check().
2433
2434
        :param revision_ids: A non-empty list of revision_ids whose ancestry
2435
             will be checked.  Typically the last revision_id of a branch.
2436
        """
2437
        return self._check(revision_ids)
2438
2439
    def _check(self, revision_ids):
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
2440
        result = check.Check(self)
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
2441
        result.check()
2442
        return result
2443
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
2444
    def _warn_if_deprecated(self):
1904.2.5 by Martin Pool
Fix format warning inside test suite and add test
2445
        global _deprecation_warning_done
2446
        if _deprecation_warning_done:
2447
            return
2448
        _deprecation_warning_done = True
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
2449
        warning("Format %s for %s is deprecated - please use 'bzr upgrade' to get better performance"
2450
                % (self._format, self.bzrdir.transport.base))
2451
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
2452
    def supports_rich_root(self):
2453
        return self._format.rich_root_data
2454
2150.2.2 by Robert Collins
Change the commit builder selected-revision-id test to use a unicode revision id where possible, leading to stricter testing of the hypothetical unicode revision id support in bzr.
2455
    def _check_ascii_revisionid(self, revision_id, method):
2456
        """Private helper for ascii-only repositories."""
2457
        # weave repositories refuse to store revisionids that are non-ascii.
2458
        if revision_id is not None:
2459
            # weaves require ascii revision ids.
2460
            if isinstance(revision_id, unicode):
2461
                try:
2462
                    revision_id.encode('ascii')
2463
                except UnicodeEncodeError:
2464
                    raise errors.NonAsciiRevisionId(method, self)
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
2465
            else:
2466
                try:
2467
                    revision_id.decode('ascii')
2468
                except UnicodeDecodeError:
2469
                    raise errors.NonAsciiRevisionId(method, self)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2470
2819.2.4 by Andrew Bennetts
Add a 'revision_graph_can_have_wrong_parents' method to repository.
2471
    def revision_graph_can_have_wrong_parents(self):
2472
        """Is it possible for this repository to have a revision graph with
2473
        incorrect parents?
2150.2.2 by Robert Collins
Change the commit builder selected-revision-id test to use a unicode revision id where possible, leading to stricter testing of the hypothetical unicode revision id support in bzr.
2474
2819.2.4 by Andrew Bennetts
Add a 'revision_graph_can_have_wrong_parents' method to repository.
2475
        If True, then this repository must also implement
2476
        _find_inconsistent_revision_parents so that check and reconcile can
2477
        check for inconsistencies before proceeding with other checks that may
2478
        depend on the revision index being consistent.
2479
        """
2480
        raise NotImplementedError(self.revision_graph_can_have_wrong_parents)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
2481
2482
2241.1.18 by mbp at sourcefrog
Restore use of deprecating delegator for old formats in bzrlib.repository.
2483
# remove these delegates a while after bzr 0.15
2484
def __make_delegated(name, from_module):
2485
    def _deprecated_repository_forwarder():
2486
        symbol_versioning.warn('%s moved to %s in bzr 0.15'
2487
            % (name, from_module),
2241.1.20 by mbp at sourcefrog
update tests for new locations of weave repos
2488
            DeprecationWarning,
2489
            stacklevel=2)
2241.1.18 by mbp at sourcefrog
Restore use of deprecating delegator for old formats in bzrlib.repository.
2490
        m = __import__(from_module, globals(), locals(), [name])
2491
        try:
2492
            return getattr(m, name)
2493
        except AttributeError:
2494
            raise AttributeError('module %s has no name %s'
2495
                    % (m, name))
2496
    globals()[name] = _deprecated_repository_forwarder
2497
2498
for _name in [
2499
        'AllInOneRepository',
2500
        'WeaveMetaDirRepository',
2501
        'PreSplitOutRepositoryFormat',
2502
        'RepositoryFormat4',
2503
        'RepositoryFormat5',
2504
        'RepositoryFormat6',
2505
        'RepositoryFormat7',
2506
        ]:
2507
    __make_delegated(_name, 'bzrlib.repofmt.weaverepo')
2508
2509
for _name in [
2510
        'KnitRepository',
2511
        'RepositoryFormatKnit',
2512
        'RepositoryFormatKnit1',
2513
        ]:
2514
    __make_delegated(_name, 'bzrlib.repofmt.knitrepo')
2515
2516
2996.2.2 by Aaron Bentley
Create install_revisions function
2517
def install_revision(repository, rev, revision_tree):
2518
    """Install all revision data into a repository."""
2519
    install_revisions(repository, [(rev, revision_tree, None)])
2520
2521
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2522
def install_revisions(repository, iterable, num_revisions=None, pb=None):
2996.2.4 by Aaron Bentley
Rename function to add_signature_text
2523
    """Install all revision data into a repository.
2524
2525
    Accepts an iterable of revision, tree, signature tuples.  The signature
2526
    may be None.
2527
    """
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2528
    repository.start_write_group()
2529
    try:
3735.2.13 by Robert Collins
Teach install_revisions to use inventory deltas when appropriate.
2530
        inventory_cache = lru_cache.LRUCache(10)
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2531
        for n, (revision, revision_tree, signature) in enumerate(iterable):
3735.2.13 by Robert Collins
Teach install_revisions to use inventory deltas when appropriate.
2532
            _install_revision(repository, revision, revision_tree, signature,
2533
                inventory_cache)
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
2534
            if pb is not None:
2535
                pb.update('Transferring revisions', n + 1, num_revisions)
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2536
    except:
2537
        repository.abort_write_group()
2592.3.101 by Robert Collins
Correctly propogate exceptions from repository.install_revisions.
2538
        raise
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2539
    else:
2540
        repository.commit_write_group()
2541
2542
3735.2.13 by Robert Collins
Teach install_revisions to use inventory deltas when appropriate.
2543
def _install_revision(repository, rev, revision_tree, signature,
2544
    inventory_cache):
2592.3.96 by Robert Collins
Merge index improvements (includes bzr.dev).
2545
    """Install all revision data into a repository."""
1185.82.84 by Aaron Bentley
Moved stuff around
2546
    present_parents = []
2547
    parent_trees = {}
2548
    for p_id in rev.parent_ids:
2549
        if repository.has_revision(p_id):
2550
            present_parents.append(p_id)
2551
            parent_trees[p_id] = repository.revision_tree(p_id)
2552
        else:
3668.5.1 by Jelmer Vernooij
Use NULL_REVISION rather than None for Repository.revision_tree().
2553
            parent_trees[p_id] = repository.revision_tree(
2554
                                     _mod_revision.NULL_REVISION)
1185.82.84 by Aaron Bentley
Moved stuff around
2555
2556
    inv = revision_tree.inventory
1910.2.51 by Aaron Bentley
Bundles now corrupt repositories
2557
    entries = inv.iter_entries()
2617.6.6 by Robert Collins
Some review feedback.
2558
    # backwards compatibility hack: skip the root id.
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
2559
    if not repository.supports_rich_root():
1910.2.60 by Aaron Bentley
Ensure that new-model revisions aren't installed into old-model repos
2560
        path, root = entries.next()
2561
        if root.revision != rev.revision_id:
1910.2.63 by Aaron Bentley
Add supports_rich_root member to repository
2562
            raise errors.IncompatibleRevision(repr(repository))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2563
    text_keys = {}
2564
    for path, ie in entries:
2565
        text_keys[(ie.file_id, ie.revision)] = ie
2566
    text_parent_map = repository.texts.get_parent_map(text_keys)
2567
    missing_texts = set(text_keys) - set(text_parent_map)
1185.82.84 by Aaron Bentley
Moved stuff around
2568
    # Add the texts that are not already present
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2569
    for text_key in missing_texts:
2570
        ie = text_keys[text_key]
2571
        text_parents = []
2572
        # FIXME: TODO: The following loop overlaps/duplicates that done by
2573
        # commit to determine parents. There is a latent/real bug here where
2574
        # the parents inserted are not those commit would do - in particular
2575
        # they are not filtered by heads(). RBC, AB
2576
        for revision, tree in parent_trees.iteritems():
2577
            if ie.file_id not in tree:
2578
                continue
2579
            parent_id = tree.inventory[ie.file_id].revision
2580
            if parent_id in text_parents:
2581
                continue
2582
            text_parents.append((ie.file_id, parent_id))
2583
        lines = revision_tree.get_file(ie.file_id).readlines()
2584
        repository.texts.add_lines(text_key, text_parents, lines)
1185.82.84 by Aaron Bentley
Moved stuff around
2585
    try:
2586
        # install the inventory
3735.2.13 by Robert Collins
Teach install_revisions to use inventory deltas when appropriate.
2587
        if repository._format._commit_inv_deltas and len(rev.parent_ids):
2588
            # Cache this inventory
2589
            inventory_cache[rev.revision_id] = inv
2590
            try:
2591
                basis_inv = inventory_cache[rev.parent_ids[0]]
2592
            except KeyError:
2593
                repository.add_inventory(rev.revision_id, inv, present_parents)
2594
            else:
3735.2.47 by Robert Collins
Move '_make_inv_delta' onto Inventory (UNTESTED).
2595
                delta = inv._make_delta(basis_inv)
3735.13.4 by John Arbash Meinel
Track down more code paths that were broken by the merge.
2596
                repository.add_inventory_by_delta(rev.parent_ids[0], delta,
3735.2.13 by Robert Collins
Teach install_revisions to use inventory deltas when appropriate.
2597
                    rev.revision_id, present_parents)
2598
        else:
2599
            repository.add_inventory(rev.revision_id, inv, present_parents)
1185.82.84 by Aaron Bentley
Moved stuff around
2600
    except errors.RevisionAlreadyPresent:
2601
        pass
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
2602
    if signature is not None:
2996.2.8 by Aaron Bentley
Fix add_signature discrepancies
2603
        repository.add_signature_text(rev.revision_id, signature)
1185.82.84 by Aaron Bentley
Moved stuff around
2604
    repository.add_revision(rev.revision_id, rev, inv)
2605
2606
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2607
class MetaDirRepository(Repository):
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
2608
    """Repositories in the new meta-dir layout.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2609
3407.2.13 by Martin Pool
Remove indirection through control_files to get transports
2610
    :ivar _transport: Transport for access to repository control files,
2611
        typically pointing to .bzr/repository.
2612
    """
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2613
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2614
    def __init__(self, _format, a_bzrdir, control_files):
2615
        super(MetaDirRepository, self).__init__(_format, a_bzrdir, control_files)
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2616
        self._transport = control_files._transport
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2617
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2618
    def is_shared(self):
2619
        """Return True if this repository is flagged as a shared repository."""
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2620
        return self._transport.has('shared-storage')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2621
2622
    @needs_write_lock
2623
    def set_make_working_trees(self, new_value):
2624
        """Set the policy flag for making working trees when creating branches.
2625
2626
        This only applies to branches that use this repository.
2627
2628
        The default is 'True'.
2629
        :param new_value: True to restore the default, False to disable making
2630
                          working trees.
2631
        """
2632
        if new_value:
2633
            try:
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2634
                self._transport.delete('no-working-trees')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2635
            except errors.NoSuchFile:
2636
                pass
2637
        else:
3407.2.5 by Martin Pool
Deprecate LockableFiles.put_utf8
2638
            self._transport.put_bytes('no-working-trees', '',
3407.2.18 by Martin Pool
BzrDir takes responsibility for default file/dir modes
2639
                mode=self.bzrdir._get_file_mode())
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2640
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2641
    def make_working_trees(self):
2642
        """Returns the policy for making working trees on new branches."""
3407.2.3 by Martin Pool
Branch and Repository use their own ._transport rather than going through .control_files
2643
        return not self._transport.has('no-working-trees')
1596.2.12 by Robert Collins
Merge and make Knit Repository use the revision store for all possible queries.
2644
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2645
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
2646
class MetaDirVersionedFileRepository(MetaDirRepository):
2647
    """Repositories in a meta-dir, that work via versioned file objects."""
2648
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2649
    def __init__(self, _format, a_bzrdir, control_files):
3316.2.5 by Robert Collins
Review feedback.
2650
        super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
2651
            control_files)
3316.2.3 by Robert Collins
Remove manual notification of transaction finishing on versioned files.
2652
2653
4032.3.1 by Robert Collins
Add a BranchFormat.network_name() method as preparation for creating branches via RPC calls.
2654
network_format_registry = registry.FormatRegistry()
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2655
"""Registry of formats indexed by their network name.
2656
2657
The network name for a repository format is an identifier that can be used when
2658
referring to formats with smart server operations. See
2659
RepositoryFormat.network_name() for more detail.
2660
"""
3990.5.1 by Andrew Bennetts
Add network_name() to RepositoryFormat.
2661
2662
4032.3.1 by Robert Collins
Add a BranchFormat.network_name() method as preparation for creating branches via RPC calls.
2663
format_registry = registry.FormatRegistry(network_format_registry)
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2664
"""Registry of formats, indexed by their BzrDirMetaFormat format string.
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2665
2666
This can contain either format instances themselves, or classes/factories that
2667
can be called to obtain one.
2668
"""
2241.1.2 by Martin Pool
change to using external Repository format registry
2669
2220.2.3 by Martin Pool
Add tag: revision namespace.
2670
2671
#####################################################################
2672
# Repository Formats
1910.2.46 by Aaron Bentley
Whitespace fix
2673
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2674
class RepositoryFormat(object):
2675
    """A repository format.
2676
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2677
    Formats provide four things:
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2678
     * An initialization routine to construct repository data on disk.
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2679
     * a optional format string which is used when the BzrDir supports
2680
       versioned children.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2681
     * an open routine which returns a Repository instance.
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2682
     * A network name for referring to the format in smart server RPC
2683
       methods.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2684
2889.1.2 by Robert Collins
Review feedback.
2685
    There is one and only one Format subclass for each on-disk format. But
2686
    there can be one Repository subclass that is used for several different
2687
    formats. The _format attribute on a Repository instance can be used to
2688
    determine the disk format.
2889.1.1 by Robert Collins
* The class ``bzrlib.repofmt.knitrepo.KnitRepository3`` has been folded into
2689
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2690
    Formats are placed in a registry by their format string for reference
2691
    during opening. These should be subclasses of RepositoryFormat for
2692
    consistency.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2693
2694
    Once a format is deprecated, just deprecate the initialize and open
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2695
    methods on the format class. Do not deprecate the object, as the
4031.3.1 by Frank Aspell
Fixing various typos
2696
    object may be created even when a repository instance hasn't been
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2697
    created.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2698
2699
    Common instance attributes:
2700
    _matchingbzrdir - the bzrdir format that the repository format was
2701
    originally written to work with. This can be used if manually
2702
    constructing a bzrdir and repository, or more commonly for test suite
3128.1.3 by Vincent Ladeuil
Since we are there s/parameteris.*/parameteriz&/.
2703
    parameterization.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2704
    """
2705
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2706
    # Set to True or False in derived classes. True indicates that the format
2707
    # supports ghosts gracefully.
2708
    supports_ghosts = None
3221.3.1 by Robert Collins
* Repository formats have a new supported-feature attribute
2709
    # Can this repository be given external locations to lookup additional
2710
    # data. Set to True or False in derived classes.
2711
    supports_external_lookups = None
3735.2.1 by Robert Collins
Add the concept of CHK lookups to Repository.
2712
    # Does this format support CHK bytestring lookups. Set to True or False in
2713
    # derived classes.
2714
    supports_chks = None
3735.2.12 by Robert Collins
Implement commit-via-deltas for split inventory repositories.
2715
    # Should commit add an inventory, or an inventory delta to the repository.
2716
    _commit_inv_deltas = True
4053.1.4 by Robert Collins
Move the fetch control attributes from Repository to RepositoryFormat.
2717
    # What order should fetch operations request streams in?
2718
    # The default is unordered as that is the cheapest for an origin to
2719
    # provide.
2720
    _fetch_order = 'unordered'
2721
    # Does this repository format use deltas that can be fetched as-deltas ?
2722
    # (E.g. knits, where the knit deltas can be transplanted intact.
2723
    # We default to False, which will ensure that enough data to get
2724
    # a full text out of any fetch stream will be grabbed.
2725
    _fetch_uses_deltas = False
2726
    # Should fetch trigger a reconcile after the fetch? Only needed for
2727
    # some repository formats that can suffer internal inconsistencies.
2728
    _fetch_reconcile = False
4183.5.1 by Robert Collins
Add RepositoryFormat.fast_deltas to signal fast delta creation.
2729
    # Does this format have < O(tree_size) delta generation. Used to hint what
2730
    # code path for commit, amongst other things.
2731
    fast_deltas = None
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
2732
1904.2.3 by Martin Pool
Give a warning on access to old repository formats
2733
    def __str__(self):
2734
        return "<%s>" % self.__class__.__name__
2735
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2736
    def __eq__(self, other):
2737
        # format objects are generally stateless
2738
        return isinstance(other, self.__class__)
2739
2100.3.35 by Aaron Bentley
equality operations on bzrdir
2740
    def __ne__(self, other):
2100.3.31 by Aaron Bentley
Merged bzr.dev (17 tests failing)
2741
        return not self == other
2742
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2743
    @classmethod
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2744
    def find_format(klass, a_bzrdir):
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2745
        """Return the format for the repository object in a_bzrdir.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2746
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2747
        This is used by bzr native formats that have a "format" file in
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2748
        the repository.  Other methods may be used by different types of
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2749
        control directory.
2750
        """
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2751
        try:
2752
            transport = a_bzrdir.get_repository_transport(None)
2753
            format_string = transport.get("format").read()
2241.1.2 by Martin Pool
change to using external Repository format registry
2754
            return format_registry.get(format_string)
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2755
        except errors.NoSuchFile:
2756
            raise errors.NoRepositoryPresent(a_bzrdir)
2757
        except KeyError:
3246.3.2 by Daniel Watkins
Modified uses of errors.UnknownFormatError.
2758
            raise errors.UnknownFormatError(format=format_string,
2759
                                            kind='repository')
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2760
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2761
    @classmethod
2241.1.2 by Martin Pool
change to using external Repository format registry
2762
    def register_format(klass, format):
2763
        format_registry.register(format.get_format_string(), format)
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2764
2765
    @classmethod
2766
    def unregister_format(klass, format):
2241.1.2 by Martin Pool
change to using external Repository format registry
2767
        format_registry.remove(format.get_format_string())
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2768
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2769
    @classmethod
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2770
    def get_default_format(klass):
2771
        """Return the current default format."""
2204.5.3 by Aaron Bentley
zap old repository default handling
2772
        from bzrlib import bzrdir
2773
        return bzrdir.format_registry.make_bzrdir('default').repository_format
2241.1.1 by Martin Pool
Change RepositoryFormat to use a Registry rather than ad-hoc dictionary
2774
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2775
    def get_format_string(self):
2776
        """Return the ASCII format string that identifies this format.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2777
2778
        Note that in pre format ?? repositories the format string is
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2779
        not permitted nor written to disk.
2780
        """
2781
        raise NotImplementedError(self.get_format_string)
2782
1624.3.19 by Olaf Conradi
New call get_format_description to give a user-friendly description of a
2783
    def get_format_description(self):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
2784
        """Return the short description for this format."""
1624.3.19 by Olaf Conradi
New call get_format_description to give a user-friendly description of a
2785
        raise NotImplementedError(self.get_format_description)
2786
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2787
    # TODO: this shouldn't be in the base class, it's specific to things that
2788
    # use weaves or knits -- mbp 20070207
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2789
    def _get_versioned_file_store(self,
2790
                                  name,
2791
                                  transport,
2792
                                  control_files,
2793
                                  prefixed=True,
2241.1.10 by Martin Pool
Remove more references to weaves from the repository.py file
2794
                                  versionedfile_class=None,
1946.2.5 by John Arbash Meinel
Make knit stores delay creation, but not control stores
2795
                                  versionedfile_kwargs={},
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2796
                                  escaped=False):
2241.1.10 by Martin Pool
Remove more references to weaves from the repository.py file
2797
        if versionedfile_class is None:
2798
            versionedfile_class = self._versionedfile_class
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2799
        weave_transport = control_files._transport.clone(name)
2800
        dir_mode = control_files._dir_mode
2801
        file_mode = control_files._file_mode
2802
        return VersionedFileStore(weave_transport, prefixed=prefixed,
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2803
                                  dir_mode=dir_mode,
2804
                                  file_mode=file_mode,
2805
                                  versionedfile_class=versionedfile_class,
1946.2.5 by John Arbash Meinel
Make knit stores delay creation, but not control stores
2806
                                  versionedfile_kwargs=versionedfile_kwargs,
1608.2.12 by Martin Pool
Store-escaping must quote uppercase characters too, so that they're safely
2807
                                  escaped=escaped)
1563.2.17 by Robert Collins
Change knits repositories to use a knit versioned file store for file texts.
2808
1534.6.1 by Robert Collins
allow API creation of shared repositories
2809
    def initialize(self, a_bzrdir, shared=False):
2810
        """Initialize a repository of this format in a_bzrdir.
2811
2812
        :param a_bzrdir: The bzrdir to put the new repository in it.
2813
        :param shared: The repository should be initialized as a sharable one.
1752.2.52 by Andrew Bennetts
Flesh out more Remote* methods needed to open and initialise remote branches/trees/repositories.
2814
        :returns: The new repository object.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2815
1534.6.1 by Robert Collins
allow API creation of shared repositories
2816
        This may raise UninitializableFormat if shared repository are not
2817
        compatible the a_bzrdir.
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2818
        """
1752.2.52 by Andrew Bennetts
Flesh out more Remote* methods needed to open and initialise remote branches/trees/repositories.
2819
        raise NotImplementedError(self.initialize)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2820
2821
    def is_supported(self):
2822
        """Is this format supported?
2823
2824
        Supported formats must be initializable and openable.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2825
        Unsupported formats may not support initialization or committing or
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2826
        some other features depending on the reason for not being supported.
2827
        """
2828
        return True
2829
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2830
    def network_name(self):
2831
        """A simple byte string uniquely identifying this format for RPC calls.
2832
2833
        MetaDir repository formats use their disk format string to identify the
2834
        repository over the wire. All in one formats such as bzr < 0.8, and
2835
        foreign formats like svn/git and hg should use some marker which is
2836
        unique and immutable.
2837
        """
2838
        raise NotImplementedError(self.network_name)
2839
1910.2.12 by Aaron Bentley
Implement knit repo format 2
2840
    def check_conversion_target(self, target_format):
2841
        raise NotImplementedError(self.check_conversion_target)
2842
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2843
    def open(self, a_bzrdir, _found=False):
2844
        """Return an instance of this format for the bzrdir a_bzrdir.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2845
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2846
        _found is a private parameter, do not use it.
2847
        """
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2848
        raise NotImplementedError(self.open)
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2849
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2850
2851
class MetaDirRepositoryFormat(RepositoryFormat):
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
2852
    """Common base class for the new repositories using the metadir layout."""
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2853
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2854
    rich_root_data = False
2323.5.17 by Martin Pool
Add supports_tree_reference to all repo formats (robert)
2855
    supports_tree_reference = False
3221.3.1 by Robert Collins
* Repository formats have a new supported-feature attribute
2856
    supports_external_lookups = False
3845.1.1 by John Arbash Meinel
Ensure that RepositoryFormat._matchingbzrdir.repository_format matches.
2857
2858
    @property
2859
    def _matchingbzrdir(self):
2860
        matching = bzrdir.BzrDirMetaFormat1()
2861
        matching.repository_format = self
2862
        return matching
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
2863
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
2864
    def __init__(self):
2865
        super(MetaDirRepositoryFormat, self).__init__()
2866
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2867
    def _create_control_files(self, a_bzrdir):
2868
        """Create the required files and the initial control_files object."""
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
2869
        # FIXME: RBC 20060125 don't peek under the covers
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2870
        # NB: no need to escape relative paths that are url safe.
2871
        repository_transport = a_bzrdir.get_repository_transport(self)
1996.3.4 by John Arbash Meinel
lazy_import bzrlib/repository.py
2872
        control_files = lockable_files.LockableFiles(repository_transport,
2873
                                'lock', lockdir.LockDir)
1553.5.61 by Martin Pool
Locks protecting LockableFiles must now be explicitly created before use.
2874
        control_files.create_lock()
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2875
        return control_files
2876
2877
    def _upload_blank_content(self, a_bzrdir, dirs, files, utf8_files, shared):
2878
        """Upload the initial blank content."""
2879
        control_files = self._create_control_files(a_bzrdir)
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2880
        control_files.lock_write()
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2881
        transport = control_files._transport
2882
        if shared == True:
2883
            utf8_files += [('shared-storage', '')]
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2884
        try:
3407.2.18 by Martin Pool
BzrDir takes responsibility for default file/dir modes
2885
            transport.mkdir_multi(dirs, mode=a_bzrdir._get_dir_mode())
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2886
            for (filename, content_stream) in files:
2887
                transport.put_file(filename, content_stream,
3407.2.18 by Martin Pool
BzrDir takes responsibility for default file/dir modes
2888
                    mode=a_bzrdir._get_file_mode())
3407.2.4 by Martin Pool
Small cleanups to initial creation of repository files
2889
            for (filename, content_bytes) in utf8_files:
2890
                transport.put_bytes_non_atomic(filename, content_bytes,
3407.2.18 by Martin Pool
BzrDir takes responsibility for default file/dir modes
2891
                    mode=a_bzrdir._get_file_mode())
1534.4.47 by Robert Collins
Split out repository into .bzr/repository
2892
        finally:
2893
            control_files.unlock()
1556.1.3 by Robert Collins
Rearrangment of Repository logic to be less type code driven, and bugfix InterRepository.missing_revision_ids
2894
3990.5.1 by Andrew Bennetts
Add network_name() to RepositoryFormat.
2895
    def network_name(self):
2896
        """Metadir formats have matching disk and network format strings."""
2897
        return self.get_format_string()
2898
2899
3990.5.3 by Robert Collins
Docs and polish on RepositoryFormat.network_name.
2900
# Pre-0.8 formats that don't have a disk format string (because they are
2901
# versioned by the matching control directory). We use the control directories
2902
# disk format string as a key for the network_name because they meet the
4031.3.1 by Frank Aspell
Fixing various typos
2903
# constraints (simple string, unique, immutable).
3990.5.1 by Andrew Bennetts
Add network_name() to RepositoryFormat.
2904
network_format_registry.register_lazy(
2905
    "Bazaar-NG branch, format 5\n",
2906
    'bzrlib.repofmt.weaverepo',
2907
    'RepositoryFormat5',
2908
)
2909
network_format_registry.register_lazy(
2910
    "Bazaar-NG branch, format 6\n",
2911
    'bzrlib.repofmt.weaverepo',
2912
    'RepositoryFormat6',
2913
)
2914
2915
# formats which have no format string are not discoverable or independently
4032.1.1 by John Arbash Meinel
Merge the removal of all trailing whitespace, and resolve conflicts.
2916
# creatable on disk, so are not registered in format_registry.  They're
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2917
# all in bzrlib.repofmt.weaverepo now.  When an instance of one of these is
2918
# needed, it's constructed directly by the BzrDir.  Non-native formats where
2919
# the repository is not separately opened are similar.
2920
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
2921
format_registry.register_lazy(
2922
    'Bazaar-NG Repository format 7',
2923
    'bzrlib.repofmt.weaverepo',
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2924
    'RepositoryFormat7'
2241.1.4 by Martin Pool
Moved old weave-based repository formats into bzrlib.repofmt.weaverepo.
2925
    )
2592.3.22 by Robert Collins
Add new experimental repository formats.
2926
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2927
format_registry.register_lazy(
2928
    'Bazaar-NG Knit Repository Format 1',
2929
    'bzrlib.repofmt.knitrepo',
2241.1.11 by Martin Pool
Get rid of RepositoryFormat*_instance objects. Instead the format
2930
    'RepositoryFormatKnit1',
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
2931
    )
2932
2241.1.5 by Martin Pool
Move KnitFormat2 into repofmt
2933
format_registry.register_lazy(
2255.2.230 by Robert Collins
Update tree format signatures to mention introducing bzr version.
2934
    'Bazaar Knit Repository Format 3 (bzr 0.15)\n',
2100.3.31 by Aaron Bentley
Merged bzr.dev (17 tests failing)
2935
    'bzrlib.repofmt.knitrepo',
2936
    'RepositoryFormatKnit3',
2937
    )
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
2938
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
2939
format_registry.register_lazy(
2940
    'Bazaar Knit Repository Format 4 (bzr 1.0)\n',
2941
    'bzrlib.repofmt.knitrepo',
2942
    'RepositoryFormatKnit4',
2943
    )
2944
2939.2.1 by Ian Clatworthy
use 'knitpack' naming instead of 'experimental' for pack formats
2945
# Pack-based formats. There is one format for pre-subtrees, and one for
2946
# post-subtrees to allow ease of testing.
3152.2.1 by Robert Collins
* A new repository format 'development' has been added. This format will
2947
# NOTE: These are experimental in 0.92. Stable in 1.0 and above
2592.3.22 by Robert Collins
Add new experimental repository formats.
2948
format_registry.register_lazy(
2939.2.6 by Ian Clatworthy
more review feedback from lifeless and poolie
2949
    'Bazaar pack repository format 1 (needs bzr 0.92)\n',
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2950
    'bzrlib.repofmt.pack_repo',
2592.3.224 by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository
2951
    'RepositoryFormatKnitPack1',
2592.3.22 by Robert Collins
Add new experimental repository formats.
2952
    )
2953
format_registry.register_lazy(
2939.2.6 by Ian Clatworthy
more review feedback from lifeless and poolie
2954
    'Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n',
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
2955
    'bzrlib.repofmt.pack_repo',
2592.3.224 by Martin Pool
Rename GraphKnitRepository etc to KnitPackRepository
2956
    'RepositoryFormatKnitPack3',
2592.3.22 by Robert Collins
Add new experimental repository formats.
2957
    )
2996.2.11 by Aaron Bentley
Implement rich-root-pack format ( #164639)
2958
format_registry.register_lazy(
2959
    'Bazaar pack repository format 1 with rich root (needs bzr 1.0)\n',
2960
    'bzrlib.repofmt.pack_repo',
2961
    'RepositoryFormatKnitPack4',
2962
    )
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2963
format_registry.register_lazy(
2964
    'Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n',
2965
    'bzrlib.repofmt.pack_repo',
2966
    'RepositoryFormatKnitPack5',
2967
    )
2968
format_registry.register_lazy(
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
2969
    'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n',
2970
    'bzrlib.repofmt.pack_repo',
2971
    'RepositoryFormatKnitPack5RichRoot',
2972
    )
2973
format_registry.register_lazy(
3549.1.6 by Martin Pool
Change stacked-subtree to stacked-rich-root
2974
    'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n',
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2975
    'bzrlib.repofmt.pack_repo',
3606.10.1 by John Arbash Meinel
Create a new --1.6-rich-root, deprecate the old one.
2976
    'RepositoryFormatKnitPack5RichRootBroken',
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2977
    )
3805.3.1 by John Arbash Meinel
Add repository 1.9 format, and update the documentation.
2978
format_registry.register_lazy(
2979
    'Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n',
2980
    'bzrlib.repofmt.pack_repo',
2981
    'RepositoryFormatKnitPack6',
2982
    )
2983
format_registry.register_lazy(
2984
    'Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n',
2985
    'bzrlib.repofmt.pack_repo',
2986
    'RepositoryFormatKnitPack6RichRoot',
2987
    )
3549.1.5 by Martin Pool
Add stable format names for stacked branches
2988
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
2989
# Development formats.
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
2990
# Obsolete but kept pending a CHK based subtree format.
3735.1.1 by Robert Collins
Add development2 formats using BTree indices.
2991
format_registry.register_lazy(
2992
    ("Bazaar development format 2 with subtree support "
2993
        "(needs bzr.dev from before 1.8)\n"),
2994
    'bzrlib.repofmt.pack_repo',
2995
    'RepositoryFormatPackDevelopment2Subtree',
2996
    )
2592.3.22 by Robert Collins
Add new experimental repository formats.
2997
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
2998
# 1.14->1.16 go below here
2999
format_registry.register_lazy(
3000
    'Bazaar development format - group compression and chk inventory'
3001
        ' (needs bzr.dev from 1.14)\n',
3002
    'bzrlib.repofmt.groupcompress_repo',
3003
    'RepositoryFormatCHK1',
3735.31.1 by John Arbash Meinel
Bring the groupcompress plugin into the brisbane-core branch.
3004
    )
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3005
1534.4.40 by Robert Collins
Add RepositoryFormats and allow bzrdir.open or create _repository to be used.
3006
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
3007
class InterRepository(InterObject):
1534.1.27 by Robert Collins
Start InterRepository with InterRepository.get.
3008
    """This class represents operations taking place between two repositories.
3009
1534.1.33 by Robert Collins
Move copy_content_into into InterRepository and InterWeaveRepo, and disable the default codepath test as we have optimised paths for all current combinations.
3010
    Its instances have methods like copy_content and fetch, and contain
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3011
    references to the source and target repositories these operations can be
1534.1.27 by Robert Collins
Start InterRepository with InterRepository.get.
3012
    carried out on.
3013
3014
    Often we will provide convenience methods on 'repository' which carry out
3015
    operations with another repository - they will always forward to
3016
    InterRepository.get(other).method_name(parameters).
3017
    """
3018
4144.2.1 by Andrew Bennetts
Always batch revisions to ask of target when doing _walk_to_common_revisions, rather than special-casing in Inter*Remote*.
3019
    _walk_to_common_revisions_batch_size = 50
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3020
    _optimisers = []
1534.1.28 by Robert Collins
Allow for optimised InterRepository selection.
3021
    """The available optimised InterRepository types."""
3022
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
3023
    @needs_write_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3024
    def copy_content(self, revision_id=None):
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
3025
        """Make a complete copy of the content in self into destination.
3026
3027
        This is a destructive operation! Do not use it on existing
3028
        repositories.
3029
3030
        :param revision_id: Only copy the content needed to construct
3031
                            revision_id and its parents.
3032
        """
3033
        try:
3034
            self.target.set_make_working_trees(self.source.make_working_trees())
3035
        except NotImplementedError:
3036
            pass
3037
        self.target.fetch(self.source, revision_id=revision_id)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3038
4110.2.23 by Martin Pool
blackbox hpss test should check repository was remotely locked
3039
    @needs_write_lock
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3040
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3041
            fetch_spec=None):
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
3042
        """Fetch the content required to construct revision_id.
3043
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
3044
        The content is copied from self.source to self.target.
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
3045
3046
        :param revision_id: if None all content is copied, if NULL_REVISION no
3047
                            content is copied.
3048
        :param pb: optional progress bar to use for progress reports. If not
3049
                   provided a default one will be created.
4065.1.1 by Robert Collins
Change the return value of fetch() to None.
3050
        :return: None.
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
3051
        """
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
3052
        from bzrlib.fetch import RepoFetcher
3053
        f = RepoFetcher(to_repository=self.target,
3054
                               from_repository=self.source,
3055
                               last_revision=revision_id,
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3056
                               fetch_spec=fetch_spec,
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
3057
                               pb=pb, find_ghosts=find_ghosts)
3172.4.4 by Robert Collins
Review feedback.
3058
3059
    def _walk_to_common_revisions(self, revision_ids):
3060
        """Walk out from revision_ids in source to revisions target has.
3061
3062
        :param revision_ids: The start point for the search.
3063
        :return: A set of revision ids.
3064
        """
4144.3.12 by Andrew Bennetts
Remove target_get_graph and target_get_parent_map attributes from InterRepository; nothing overrides them anymore.
3065
        target_graph = self.target.get_graph()
1551.19.41 by Aaron Bentley
Accelerate no-op pull
3066
        revision_ids = frozenset(revision_ids)
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3067
        # Fast path for the case where all the revisions are already in the
3068
        # target repo.
3069
        # (Although this does incur an extra round trip for the
3070
        # fairly common case where the target doesn't already have the revision
3071
        # we're pushing.)
1551.19.41 by Aaron Bentley
Accelerate no-op pull
3072
        if set(target_graph.get_parent_map(revision_ids)) == revision_ids:
3073
            return graph.SearchResult(revision_ids, set(), 0, set())
3172.4.4 by Robert Collins
Review feedback.
3074
        missing_revs = set()
1551.19.41 by Aaron Bentley
Accelerate no-op pull
3075
        source_graph = self.source.get_graph()
3172.4.4 by Robert Collins
Review feedback.
3076
        # ensure we don't pay silly lookup costs.
1551.19.41 by Aaron Bentley
Accelerate no-op pull
3077
        searcher = source_graph._make_breadth_first_searcher(revision_ids)
3172.4.4 by Robert Collins
Review feedback.
3078
        null_set = frozenset([_mod_revision.NULL_REVISION])
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3079
        searcher_exhausted = False
3172.4.4 by Robert Collins
Review feedback.
3080
        while True:
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3081
            next_revs = set()
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3082
            ghosts = set()
3083
            # Iterate the searcher until we have enough next_revs
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3084
            while len(next_revs) < self._walk_to_common_revisions_batch_size:
3085
                try:
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3086
                    next_revs_part, ghosts_part = searcher.next_with_ghosts()
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3087
                    next_revs.update(next_revs_part)
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3088
                    ghosts.update(ghosts_part)
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3089
                except StopIteration:
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3090
                    searcher_exhausted = True
3452.2.6 by Andrew Bennetts
Batch get_parent_map calls in InterPackToRemotePack._walk_to_common_revisions to
3091
                    break
3731.4.3 by Andrew Bennetts
Rework ghost checking in _walk_to_common_revisions.
3092
            # If there are ghosts in the source graph, and the caller asked for
3093
            # them, make sure that they are present in the target.
3731.4.5 by Andrew Bennetts
Clarify the code slightly.
3094
            # We don't care about other ghosts as we can't fetch them and
3095
            # haven't been asked to.
3096
            ghosts_to_check = set(revision_ids.intersection(ghosts))
3097
            revs_to_get = set(next_revs).union(ghosts_to_check)
3098
            if revs_to_get:
3099
                have_revs = set(target_graph.get_parent_map(revs_to_get))
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3100
                # we always have NULL_REVISION present.
3731.4.5 by Andrew Bennetts
Clarify the code slightly.
3101
                have_revs = have_revs.union(null_set)
3102
                # Check if the target is missing any ghosts we need.
3731.4.3 by Andrew Bennetts
Rework ghost checking in _walk_to_common_revisions.
3103
                ghosts_to_check.difference_update(have_revs)
3104
                if ghosts_to_check:
3105
                    # One of the caller's revision_ids is a ghost in both the
3106
                    # source and the target.
3107
                    raise errors.NoSuchRevision(
3108
                        self.source, ghosts_to_check.pop())
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3109
                missing_revs.update(next_revs - have_revs)
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
3110
                # Because we may have walked past the original stop point, make
3111
                # sure everything is stopped
3112
                stop_revs = searcher.find_seen_ancestors(have_revs)
3113
                searcher.stop_searching_any(stop_revs)
3731.4.2 by Andrew Bennetts
Move ghost check out of the inner loop.
3114
            if searcher_exhausted:
3172.4.4 by Robert Collins
Review feedback.
3115
                break
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3116
        return searcher.get_result()
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
3117
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3118
    @needs_read_lock
3119
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
3120
        """Return the revision ids that source has that target does not.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3121
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3122
        :param revision_id: only return revision ids included by this
3123
                            revision_id.
3124
        :param find_ghosts: If True find missing revisions in deep history
3125
            rather than just finding the surface difference.
3126
        :return: A bzrlib.graph.SearchResult.
3127
        """
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3128
        # stop searching at found target revisions.
3129
        if not find_ghosts and revision_id is not None:
3172.4.4 by Robert Collins
Review feedback.
3130
            return self._walk_to_common_revisions([revision_id])
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3131
        # generic, possibly worst case, slow code path.
3132
        target_ids = set(self.target.all_revision_ids())
3133
        if revision_id is not None:
3134
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
3135
            if source_ids[0] is not None:
3136
                raise AssertionError()
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3137
            source_ids.pop(0)
3138
        else:
3139
            source_ids = self.source.all_revision_ids()
3140
        result_set = set(source_ids).difference(target_ids)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3141
        return self.source.revision_ids_to_search_result(result_set)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3142
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3143
    @staticmethod
3144
    def _same_model(source, target):
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
3145
        """True if source and target have the same data representation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3146
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
3147
        Note: this is always called on the base class; overriding it in a
3148
        subclass will have no effect.
3149
        """
3150
        try:
3151
            InterRepository._assert_same_model(source, target)
3152
            return True
3153
        except errors.IncompatibleRepositories, e:
3154
            return False
3155
3156
    @staticmethod
3157
    def _assert_same_model(source, target):
3158
        """Raise an exception if two repositories do not use the same model.
3159
        """
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3160
        if source.supports_rich_root() != target.supports_rich_root():
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
3161
            raise errors.IncompatibleRepositories(source, target,
3162
                "different rich-root support")
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3163
        if source._serializer != target._serializer:
3582.1.2 by Martin Pool
Default InterRepository.fetch raises IncompatibleRepositories
3164
            raise errors.IncompatibleRepositories(source, target,
3165
                "different serializers")
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3166
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3167
3168
class InterSameDataRepository(InterRepository):
3169
    """Code for converting between repositories that represent the same data.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3170
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3171
    Data format and model must match for this to work.
3172
    """
3173
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3174
    @classmethod
2241.1.7 by Martin Pool
rename method
3175
    def _get_repo_format_to_test(self):
2814.1.1 by Robert Collins
* Pushing, pulling and branching branches with subtree references was not
3176
        """Repository format for testing with.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3177
2814.1.1 by Robert Collins
* Pushing, pulling and branching branches with subtree references was not
3178
        InterSameData can pull from subtree to subtree and from non-subtree to
3179
        non-subtree, so we test this with the richest repository format.
3180
        """
3181
        from bzrlib.repofmt import knitrepo
3182
        return knitrepo.RepositoryFormatKnit3()
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3183
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
3184
    @staticmethod
3185
    def is_compatible(source, target):
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3186
        return InterRepository._same_model(source, target)
1910.2.14 by Aaron Bentley
Fail when trying to use interrepository on Knit2 and Knit1
3187
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3188
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3189
class InterWeaveRepo(InterSameDataRepository):
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
3190
    """Optimised code paths between Weave based repositories.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3191
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
3192
    This should be in bzrlib/repofmt/weaverepo.py but we have not yet
3193
    implemented lazy inter-object optimisation.
3194
    """
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3195
2241.1.13 by Martin Pool
Re-register InterWeaveRepo, fix test integration, add test for it
3196
    @classmethod
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3197
    def _get_repo_format_to_test(self):
3198
        from bzrlib.repofmt import weaverepo
3199
        return weaverepo.RepositoryFormat7()
3200
3201
    @staticmethod
3202
    def is_compatible(source, target):
3203
        """Be compatible with known Weave formats.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3204
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3205
        We don't test for the stores being of specific types because that
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3206
        could lead to confusing results, and there is no need to be
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3207
        overly general.
3208
        """
3209
        from bzrlib.repofmt.weaverepo import (
3210
                RepositoryFormat5,
3211
                RepositoryFormat6,
3212
                RepositoryFormat7,
3213
                )
3214
        try:
3215
            return (isinstance(source._format, (RepositoryFormat5,
3216
                                                RepositoryFormat6,
3217
                                                RepositoryFormat7)) and
3218
                    isinstance(target._format, (RepositoryFormat5,
3219
                                                RepositoryFormat6,
3220
                                                RepositoryFormat7)))
3221
        except AttributeError:
3222
            return False
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3223
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3224
    @needs_write_lock
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3225
    def copy_content(self, revision_id=None):
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3226
        """See InterRepository.copy_content()."""
3227
        # weave specific optimised path:
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3228
        try:
3229
            self.target.set_make_working_trees(self.source.make_working_trees())
3349.1.2 by Aaron Bentley
Change ValueError to RepositoryUpgradeRequired
3230
        except (errors.RepositoryUpgradeRequired, NotImplemented):
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3231
            pass
3232
        # FIXME do not peek!
3407.2.14 by Martin Pool
Remove more cases of getting transport via control_files
3233
        if self.source._transport.listable():
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3234
            pb = ui.ui_factory.nested_progress_bar()
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3235
            try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3236
                self.target.texts.insert_record_stream(
3237
                    self.source.texts.get_record_stream(
3238
                        self.source.texts.keys(), 'topological', False))
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3239
                pb.update('copying inventory', 0, 1)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3240
                self.target.inventories.insert_record_stream(
3241
                    self.source.inventories.get_record_stream(
3242
                        self.source.inventories.keys(), 'topological', False))
3243
                self.target.signatures.insert_record_stream(
3244
                    self.source.signatures.get_record_stream(
3245
                        self.source.signatures.keys(),
3246
                        'unordered', True))
3247
                self.target.revisions.insert_record_stream(
3248
                    self.source.revisions.get_record_stream(
3249
                        self.source.revisions.keys(),
3250
                        'topological', True))
2387.1.1 by Robert Collins
Remove the --basis parameter to clone etc. (Robert Collins)
3251
            finally:
3252
                pb.finished()
3253
        else:
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3254
            self.target.fetch(self.source, revision_id=revision_id)
3255
3256
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3257
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3258
        """See InterRepository.missing_revision_ids()."""
3259
        # we want all revisions to satisfy revision_id in source.
3260
        # but we don't want to stat every file here and there.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3261
        # we want then, all revisions other needs to satisfy revision_id
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3262
        # checked, but not those that we have locally.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3263
        # so the first thing is to get a subset of the revisions to
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3264
        # satisfy revision_id in source, and then eliminate those that
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3265
        # we do already have.
4031.3.1 by Frank Aspell
Fixing various typos
3266
        # this is slow on high latency connection to self, but as this
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3267
        # disk format scales terribly for push anyway due to rewriting
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3268
        # inventory.weave, this is considered acceptable.
3269
        # - RBC 20060209
3270
        if revision_id is not None:
3271
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
3272
            if source_ids[0] is not None:
3273
                raise AssertionError()
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3274
            source_ids.pop(0)
3275
        else:
3276
            source_ids = self.source._all_possible_ids()
3277
        source_ids_set = set(source_ids)
3278
        # source_ids is the worst possible case we may need to pull.
3279
        # now we want to filter source_ids against what we actually
3280
        # have in target, but don't try to check for existence where we know
3281
        # we do not have a revision as that would be pointless.
3282
        target_ids = set(self.target._all_possible_ids())
3283
        possibly_present_revisions = target_ids.intersection(source_ids_set)
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3284
        actually_present_revisions = set(
3285
            self.target._eliminate_revisions_not_present(possibly_present_revisions))
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3286
        required_revisions = source_ids_set.difference(actually_present_revisions)
3287
        if revision_id is not None:
3288
            # we used get_ancestry to determine source_ids then we are assured all
3289
            # revisions referenced are present as they are installed in topological order.
3290
            # and the tip revision was validated by get_ancestry.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3291
            result_set = required_revisions
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3292
        else:
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3293
            # if we just grabbed the possibly available ids, then
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3294
            # we only have an estimate of whats available and need to validate
3295
            # that against the revision records.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3296
            result_set = set(
3297
                self.source._eliminate_revisions_not_present(required_revisions))
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3298
        return self.source.revision_ids_to_search_result(result_set)
2241.1.12 by Martin Pool
Restore InterWeaveRepo
3299
3300
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3301
class InterKnitRepo(InterSameDataRepository):
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3302
    """Optimised code paths between Knit based repositories."""
3303
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3304
    @classmethod
2241.1.7 by Martin Pool
rename method
3305
    def _get_repo_format_to_test(self):
2241.1.6 by Martin Pool
Move Knit repositories into the submodule bzrlib.repofmt.knitrepo and
3306
        from bzrlib.repofmt import knitrepo
3307
        return knitrepo.RepositoryFormatKnit1()
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3308
3309
    @staticmethod
3310
    def is_compatible(source, target):
3311
        """Be compatible with known Knit formats.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3312
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
3313
        We don't test for the stores being of specific types because that
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3314
        could lead to confusing results, and there is no need to be
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3315
        overly general.
3316
        """
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3317
        from bzrlib.repofmt.knitrepo import RepositoryFormatKnit
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3318
        try:
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3319
            are_knits = (isinstance(source._format, RepositoryFormatKnit) and
3320
                isinstance(target._format, RepositoryFormatKnit))
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3321
        except AttributeError:
3322
            return False
2592.3.28 by Robert Collins
Make InterKnitOptimiser be used between any same-model knit repository.
3323
        return are_knits and InterRepository._same_model(source, target)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3324
3325
    @needs_read_lock
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3326
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3327
        """See InterRepository.missing_revision_ids()."""
3328
        if revision_id is not None:
3329
            source_ids = self.source.get_ancestry(revision_id)
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
3330
            if source_ids[0] is not None:
3331
                raise AssertionError()
1668.1.14 by Martin Pool
merge olaf - InvalidRevisionId fixes
3332
            source_ids.pop(0)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3333
        else:
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
3334
            source_ids = self.source.all_revision_ids()
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3335
        source_ids_set = set(source_ids)
3336
        # source_ids is the worst possible case we may need to pull.
3337
        # now we want to filter source_ids against what we actually
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
3338
        # have in target, but don't try to check for existence where we know
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3339
        # we do not have a revision as that would be pointless.
2850.3.1 by Robert Collins
Move various weave specific code out of the base Repository class to weaverepo.py.
3340
        target_ids = set(self.target.all_revision_ids())
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3341
        possibly_present_revisions = target_ids.intersection(source_ids_set)
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3342
        actually_present_revisions = set(
3343
            self.target._eliminate_revisions_not_present(possibly_present_revisions))
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3344
        required_revisions = source_ids_set.difference(actually_present_revisions)
3345
        if revision_id is not None:
3346
            # we used get_ancestry to determine source_ids then we are assured all
3347
            # revisions referenced are present as they are installed in topological order.
3348
            # and the tip revision was validated by get_ancestry.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3349
            result_set = required_revisions
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3350
        else:
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3351
            # if we just grabbed the possibly available ids, then
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3352
            # we only have an estimate of whats available and need to validate
3353
            # that against the revision records.
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3354
            result_set = set(
3355
                self.source._eliminate_revisions_not_present(required_revisions))
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3356
        return self.source.revision_ids_to_search_result(result_set)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3357
1910.2.17 by Aaron Bentley
Get fetching from 1 to 2 under test
3358
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3359
class InterPackRepo(InterSameDataRepository):
3360
    """Optimised code paths between Pack based repositories."""
3361
3362
    @classmethod
3363
    def _get_repo_format_to_test(self):
3364
        from bzrlib.repofmt import pack_repo
4257.3.1 by Andrew Bennetts
Add failing test.
3365
        return pack_repo.RepositoryFormatKnitPack6RichRoot()
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3366
3367
    @staticmethod
3368
    def is_compatible(source, target):
3369
        """Be compatible with known Pack formats.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3370
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3371
        We don't test for the stores being of specific types because that
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3372
        could lead to confusing results, and there is no need to be
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3373
        overly general.
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
3374
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3375
        InterPackRepo does not support CHK based repositories.
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3376
        """
3377
        from bzrlib.repofmt.pack_repo import RepositoryFormatPack
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3378
        from bzrlib.repofmt.groupcompress_repo import RepositoryFormatCHK1
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3379
        try:
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
3380
            are_packs = (isinstance(source._format, RepositoryFormatPack) and
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3381
                isinstance(target._format, RepositoryFormatPack))
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3382
            not_packs = (isinstance(source._format, RepositoryFormatCHK1) or
3383
                isinstance(target._format, RepositoryFormatCHK1))
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3384
        except AttributeError:
3385
            return False
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3386
        if not_packs or not are_packs:
3735.31.4 by John Arbash Meinel
Continue trying to reduce the friction for the groupcompress repo formats.
3387
            return False
4241.6.8 by Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil
Add --development6-rich-root, disabling the legacy and unneeded development2 format, and activating the tests for CHK features disabled pending this format. (Robert Collins, John Arbash Meinel, Ian Clatworthy, Vincent Ladeuil)
3388
        return InterRepository._same_model(source, target)
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3389
3390
    @needs_write_lock
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3391
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3392
            fetch_spec=None):
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3393
        """See InterRepository.fetch()."""
3582.2.1 by Jonathan Lange
Fix up problems with fetching revisions. Almost entirely abentley's work.
3394
        if (len(self.source._fallback_repositories) > 0 or
3395
            len(self.target._fallback_repositories) > 0):
3582.1.13 by Martin Pool
Better comment about fetching to or from stacked repositories
3396
            # The pack layer is not aware of fallback repositories, so when
3397
            # fetching from a stacked repository or into a stacked repository
3398
            # we use the generic fetch logic which uses the VersionedFiles
3399
            # attributes on repository.
3565.3.3 by Robert Collins
* Fetching data between repositories that have the same model but no
3400
            from bzrlib.fetch import RepoFetcher
3401
            fetcher = RepoFetcher(self.target, self.source, revision_id,
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3402
                    pb, find_ghosts, fetch_spec=fetch_spec)
3403
        if fetch_spec is not None:
4070.9.16 by Andrew Bennetts
Fix small error introduced when doing review tweaks.
3404
            if len(list(fetch_spec.heads)) != 1:
3405
                raise AssertionError(
3406
                    "InterPackRepo.fetch doesn't support "
3407
                    "fetching multiple heads yet.")
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
3408
            revision_id = list(fetch_spec.heads)[0]
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3409
            fetch_spec = None
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3410
        if revision_id is None:
3411
            # TODO:
3412
            # everything to do - use pack logic
3413
            # to fetch from all packs to one without
2592.3.93 by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch.
3414
            # inventory parsing etc, IFF nothing to be copied is in the target.
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3415
            # till then:
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3416
            source_revision_ids = frozenset(self.source.all_revision_ids())
3417
            revision_ids = source_revision_ids - \
4144.3.12 by Andrew Bennetts
Remove target_get_graph and target_get_parent_map attributes from InterRepository; nothing overrides them anymore.
3418
                frozenset(self.target.get_parent_map(source_revision_ids))
1551.19.36 by Aaron Bentley
Prevent fetch all from causing pack collisions
3419
            revision_keys = [(revid,) for revid in revision_ids]
4144.3.11 by Andrew Bennetts
Remove InterPackToRemotePack too.
3420
            index = self.target._pack_collection.revision_index.combined_index
1551.19.36 by Aaron Bentley
Prevent fetch all from causing pack collisions
3421
            present_revision_ids = set(item[1][0] for item in
3422
                index.iter_entries(revision_keys))
3423
            revision_ids = set(revision_ids) - present_revision_ids
2592.3.93 by Robert Collins
Steps toward filtering revisions/inventories/texts during fetch.
3424
            # implementing the TODO will involve:
3425
            # - detecting when all of a pack is selected
3426
            # - avoiding as much as possible pre-selection, so the
3427
            # more-core routines such as create_pack_from_packs can filter in
3428
            # a just-in-time fashion. (though having a HEADS list on a
3429
            # repository might make this a lot easier, because we could
3430
            # sensibly detect 'new revisions' without doing a full index scan.
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
3431
        elif _mod_revision.is_null(revision_id):
3432
            # nothing to do:
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
3433
            return (0, [])
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3434
        else:
3435
            try:
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3436
                revision_ids = self.search_missing_revision_ids(revision_id,
3437
                    find_ghosts=find_ghosts).get_keys()
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3438
            except errors.NoSuchRevision:
3439
                raise errors.InstallFailed([revision_id])
1551.19.41 by Aaron Bentley
Accelerate no-op pull
3440
            if len(revision_ids) == 0:
3441
                return (0, [])
3452.2.2 by Andrew Bennetts
Experimental PackRepository.{check_references,autopack} RPCs.
3442
        return self._pack(self.source, self.target, revision_ids)
3452.2.1 by Andrew Bennetts
An experimental InterRepo for remote packs.
3443
3444
    def _pack(self, source, target, revision_ids):
3445
        from bzrlib.repofmt.pack_repo import Packer
3446
        packs = source._pack_collection.all_packs()
4144.3.11 by Andrew Bennetts
Remove InterPackToRemotePack too.
3447
        pack = Packer(self.target._pack_collection, packs, '.fetch',
2951.1.2 by Robert Collins
Partial refactoring of pack_repo to create a Packer object for packing.
3448
            revision_ids).pack()
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
3449
        if pack is not None:
4144.3.11 by Andrew Bennetts
Remove InterPackToRemotePack too.
3450
            self.target._pack_collection._save_pack_names()
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
3451
            copied_revs = pack.get_revision_count()
2592.3.108 by Robert Collins
Autopack after pack to pack fetching too.
3452
            # Trigger an autopack. This may duplicate effort as we've just done
3453
            # a pack creation, but for now it is simpler to think about as
3454
            # 'upload data, then repack if needed'.
4144.3.11 by Andrew Bennetts
Remove InterPackToRemotePack too.
3455
            self.target._pack_collection.autopack()
3801.1.14 by Andrew Bennetts
Reduce duplication in InterPackToRemotePack.
3456
            return (copied_revs, [])
2592.3.91 by Robert Collins
Incrementally closing in on a correct fetch for packs.
3457
        else:
3010.1.5 by Robert Collins
Test that missing_revision_ids handles the case of the source not having the requested revision correctly with and without find_ghosts.
3458
            return (0, [])
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3459
3460
    @needs_read_lock
4070.9.3 by Andrew Bennetts
Fix a bug, remove some cruft, reduce some ratchets.
3461
    def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3462
        """See InterRepository.missing_revision_ids().
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3463
3172.4.1 by Robert Collins
* Fetching via bzr+ssh will no longer fill ghosts by default (this is
3464
        :param find_ghosts: Find ghosts throughout the ancestry of
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3465
            revision_id.
3466
        """
4070.9.3 by Andrew Bennetts
Fix a bug, remove some cruft, reduce some ratchets.
3467
        if not find_ghosts and revision_id is not None:
3468
            return self._walk_to_common_revisions([revision_id])
2949.1.2 by Robert Collins
* Fetch with pack repositories will no longer read the entire history graph.
3469
        elif revision_id is not None:
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3470
            # Find ghosts: search for revisions pointing from one repository to
3221.18.1 by Ian Clatworthy
tweaks by ianc during review
3471
            # the other, and vice versa, anywhere in the history of revision_id.
4144.3.12 by Andrew Bennetts
Remove target_get_graph and target_get_parent_map attributes from InterRepository; nothing overrides them anymore.
3472
            graph = self.target.get_graph(other_repository=self.source)
3221.13.7 by Ian Clatworthy
ensure NoSuchRevision raised when needed in InterPackRepo
3473
            searcher = graph._make_breadth_first_searcher([revision_id])
3474
            found_ids = set()
3475
            while True:
3476
                try:
3477
                    next_revs, ghosts = searcher.next_with_ghosts()
3478
                except StopIteration:
3479
                    break
3480
                if revision_id in ghosts:
3481
                    raise errors.NoSuchRevision(self.source, revision_id)
3482
                found_ids.update(next_revs)
3483
                found_ids.update(ghosts)
3484
            found_ids = frozenset(found_ids)
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3485
            # Double query here: should be able to avoid this by changing the
3486
            # graph api further.
3487
            result_set = found_ids - frozenset(
4144.3.12 by Andrew Bennetts
Remove target_get_graph and target_get_parent_map attributes from InterRepository; nothing overrides them anymore.
3488
                self.target.get_parent_map(found_ids))
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3489
        else:
2592.3.151 by Robert Collins
Use the revision index, not the inventory index, for missing and fetch operations.
3490
            source_ids = self.source.all_revision_ids()
3221.13.2 by Robert Collins
Add a shallow parameter to bzrdir.sprout, which involved fixing a lateny bug in pack to pack fetching with ghost discovery.
3491
            # source_ids is the worst possible case we may need to pull.
3492
            # now we want to filter source_ids against what we actually
3493
            # have in target, but don't try to check for existence where we know
3494
            # we do not have a revision as that would be pointless.
3495
            target_ids = set(self.target.all_revision_ids())
3496
            result_set = set(source_ids).difference(target_ids)
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3497
        return self.source.revision_ids_to_search_result(result_set)
2592.3.88 by Robert Collins
Move Pack repository logic to bzrlib.repofmt.pack_repo.
3498
3499
3735.2.177 by John Arbash Meinel
InterDifferingSerializer inherits directly from InterRepository.
3500
class InterDifferingSerializer(InterRepository):
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3501
3502
    @classmethod
3503
    def _get_repo_format_to_test(self):
3504
        return None
3505
3506
    @staticmethod
3507
    def is_compatible(source, target):
3508
        """Be compatible with Knit2 source and Knit3 target"""
3735.2.178 by John Arbash Meinel
Fix another 2 failing tests.
3509
        # This is redundant with format.check_conversion_target(), however that
3510
        # raises an exception, and we just want to say "False" as in we won't
3511
        # support converting between these formats.
3512
        if source.supports_rich_root() and not target.supports_rich_root():
3513
            return False
3514
        if (source._format.supports_tree_reference
3735.2.179 by John Arbash Meinel
Fix a trivial typo.
3515
            and not target._format.supports_tree_reference):
3735.2.178 by John Arbash Meinel
Fix another 2 failing tests.
3516
            return False
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3517
        return True
3518
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3519
    def _get_delta_for_revision(self, tree, parent_ids, basis_id, cache):
3520
        """Get the best delta and base for this revision.
3521
3522
        :return: (basis_id, delta)
3523
        """
3524
        possible_trees = [(parent_id, cache[parent_id])
3525
                          for parent_id in parent_ids
3526
                           if parent_id in cache]
3527
        if len(possible_trees) == 0:
3528
            # There either aren't any parents, or the parents aren't in the
3529
            # cache, so just use the last converted tree
3530
            possible_trees.append((basis_id, cache[basis_id]))
3531
        deltas = []
3532
        for basis_id, basis_tree in possible_trees:
3533
            delta = tree.inventory._make_delta(basis_tree.inventory)
3534
            deltas.append((len(delta), basis_id, delta))
3535
        deltas.sort()
3536
        return deltas[0][1:]
3537
3735.2.181 by John Arbash Meinel
With the 'rich-root-stream' code, we also need to handle ghost parents appropriately.
3538
    def _get_parent_keys(self, root_key, parent_map):
3539
        """Get the parent keys for a given root id."""
3540
        root_id, rev_id = root_key
3541
        # Include direct parents of the revision, but only if they used
3542
        # the same root_id.
3543
        parent_keys = []
3544
        for parent_id in parent_map[rev_id]:
3545
            if parent_id == _mod_revision.NULL_REVISION:
3546
                continue
3547
            if parent_id not in self._revision_id_to_root_id:
3548
                # We probably didn't read this revision, go spend the
3549
                # extra effort to actually check
3550
                try:
3551
                    tree = self.source.revision_tree(parent_id)
3552
                except errors.NoSuchRevision:
3553
                    # Ghost, fill out _revision_id_to_root_id in case we
3554
                    # encounter this again.
3555
                    # But set parent_root_id to None since we don't really know
3556
                    parent_root_id = None
3557
                else:
3558
                    parent_root_id = tree.get_root_id()
3559
                self._revision_id_to_root_id[parent_id] = None
3560
            else:
3561
                parent_root_id = self._revision_id_to_root_id[parent_id]
3562
            if root_id == parent_root_id or parent_root_id is None:
3563
                parent_keys.append((root_id, parent_id))
3564
        return tuple(parent_keys)
3565
3566
    def _new_root_data_stream(self, root_keys_to_create, parent_map):
3567
        for root_key in root_keys_to_create:
3568
            parent_keys = self._get_parent_keys(root_key, parent_map)
3569
            yield versionedfile.FulltextContentFactory(root_key,
3570
                parent_keys, None, '')
3571
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3572
    def _fetch_batch(self, revision_ids, basis_id, cache):
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3573
        """Fetch across a few revisions.
3574
3575
        :param revision_ids: The revisions to copy
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3576
        :param basis_id: The revision_id of a tree that must be in cache, used
3577
            as a basis for delta when no other base is available
3578
        :param cache: A cache of RevisionTrees that we can use.
3579
        :return: The revision_id of the last converted tree. The RevisionTree
3580
            for it will be in cache
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3581
        """
3582
        # Walk though all revisions; get inventory deltas, copy referenced
3583
        # texts that delta references, insert the delta, revision and
3584
        # signature.
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3585
        root_keys_to_create = set()
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3586
        text_keys = set()
3587
        pending_deltas = []
3588
        pending_revisions = []
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3589
        parent_map = self.source.get_parent_map(revision_ids)
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3590
        for tree in self.source.revision_trees(revision_ids):
3591
            current_revision_id = tree.get_revision_id()
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3592
            parent_ids = parent_map.get(current_revision_id, ())
3593
            basis_id, delta = self._get_delta_for_revision(tree, parent_ids,
3594
                                                           basis_id, cache)
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3595
            if self._converting_to_rich_root:
3596
                self._revision_id_to_root_id[current_revision_id] = \
3597
                    tree.get_root_id()
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3598
            # Find text entries that need to be copied
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3599
            for old_path, new_path, file_id, entry in delta:
3600
                if new_path is not None:
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3601
                    if not new_path:
3602
                        # This is the root
3603
                        if not self.target.supports_rich_root():
3604
                            # The target doesn't support rich root, so we don't
3605
                            # copy
3606
                            continue
3607
                        if self._converting_to_rich_root:
3735.31.13 by John Arbash Meinel
A couple typo/etc fixes for the InterDifferingSerializer rich-root conversions.
3608
                            # This can't be copied normally, we have to insert
3609
                            # it specially
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3610
                            root_keys_to_create.add((file_id, entry.revision))
3735.31.13 by John Arbash Meinel
A couple typo/etc fixes for the InterDifferingSerializer rich-root conversions.
3611
                            continue
3735.2.135 by Robert Collins
Permit fetching bzr.dev [deal with inconsistent inventories.]
3612
                    text_keys.add((file_id, entry.revision))
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3613
            revision = self.source.get_revision(current_revision_id)
3614
            pending_deltas.append((basis_id, delta,
3615
                current_revision_id, revision.parent_ids))
3616
            pending_revisions.append(revision)
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3617
            cache[current_revision_id] = tree
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3618
            basis_id = current_revision_id
3619
        # Copy file texts
3620
        from_texts = self.source.texts
3621
        to_texts = self.target.texts
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3622
        if root_keys_to_create:
3735.2.181 by John Arbash Meinel
With the 'rich-root-stream' code, we also need to handle ghost parents appropriately.
3623
            root_stream = self._new_root_data_stream(root_keys_to_create,
3624
                                                     parent_map)
3625
            to_texts.insert_record_stream(root_stream)
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3626
        to_texts.insert_record_stream(from_texts.get_record_stream(
4053.1.4 by Robert Collins
Move the fetch control attributes from Repository to RepositoryFormat.
3627
            text_keys, self.target._format._fetch_order,
3628
            not self.target._format._fetch_uses_deltas))
4257.3.9 by Andrew Bennetts
Add fix to InterDifferingSerializer too, although it's pretty ugly.
3629
        # insert inventory deltas
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3630
        for delta in pending_deltas:
3631
            self.target.add_inventory_by_delta(*delta)
4257.3.9 by Andrew Bennetts
Add fix to InterDifferingSerializer too, although it's pretty ugly.
3632
        if self.target._fallback_repositories:
4257.3.10 by Andrew Bennetts
Expand comment a little.
3633
            # Make sure this stacked repository has all the parent inventories
3634
            # for the new revisions that we are about to insert.  We do this
3635
            # before adding the revisions so that no revision is added until
3636
            # all the inventories it may depend on are added.
4257.3.9 by Andrew Bennetts
Add fix to InterDifferingSerializer too, although it's pretty ugly.
3637
            parent_ids = set()
3638
            revision_ids = set()
3639
            for revision in pending_revisions:
3640
                revision_ids.add(revision.revision_id)
3641
                parent_ids.update(revision.parent_ids)
3642
            parent_ids.difference_update(revision_ids)
3643
            parent_ids.discard(_mod_revision.NULL_REVISION)
3644
            parent_map = self.source.get_parent_map(parent_ids)
3645
            for parent_tree in self.source.revision_trees(parent_ids):
3646
                basis_id, delta = self._get_delta_for_revision(tree, parent_ids, basis_id, cache)
3647
                current_revision_id = parent_tree.get_revision_id()
3648
                parents_parents = parent_map[current_revision_id]
3649
                self.target.add_inventory_by_delta(
3650
                    basis_id, delta, current_revision_id, parents_parents)
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3651
        # insert signatures and revisions
3652
        for revision in pending_revisions:
3653
            try:
3654
                signature = self.source.get_signature_text(
3655
                    revision.revision_id)
3656
                self.target.add_signature_text(revision.revision_id,
3657
                    signature)
3658
            except errors.NoSuchRevision:
3659
                pass
3660
            self.target.add_revision(revision.revision_id, revision)
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3661
        return basis_id
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3662
3663
    def _fetch_all_revisions(self, revision_ids, pb):
3664
        """Fetch everything for the list of revisions.
3665
3666
        :param revision_ids: The list of revisions to fetch. Must be in
3667
            topological order.
3668
        :param pb: A ProgressBar
3669
        :return: None
3670
        """
3671
        basis_id, basis_tree = self._get_basis(revision_ids[0])
3672
        batch_size = 100
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3673
        cache = lru_cache.LRUCache(100)
3674
        cache[basis_id] = basis_tree
3675
        del basis_tree # We don't want to hang on to it here
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3676
        for offset in range(0, len(revision_ids), batch_size):
3677
            self.target.start_write_group()
3678
            try:
3679
                pb.update('Transferring revisions', offset,
3879.2.13 by John Arbash Meinel
There was a test that asserted we called pb.update() with the last revision.
3680
                          len(revision_ids))
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3681
                batch = revision_ids[offset:offset+batch_size]
4017.4.1 by John Arbash Meinel
Change the generic fetch logic to improve delta selection.
3682
                basis_id = self._fetch_batch(batch, basis_id, cache)
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3683
            except:
3684
                self.target.abort_write_group()
3685
                raise
3686
            else:
3687
                self.target.commit_write_group()
3879.2.13 by John Arbash Meinel
There was a test that asserted we called pb.update() with the last revision.
3688
        pb.update('Transferring revisions', len(revision_ids),
3689
                  len(revision_ids))
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3690
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3691
    @needs_write_lock
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3692
    def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3693
            fetch_spec=None):
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3694
        """See InterRepository.fetch()."""
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
3695
        if fetch_spec is not None:
3696
            raise AssertionError("Not implemented yet...")
3735.31.8 by John Arbash Meinel
Some work on rich-root support.
3697
        if (not self.source.supports_rich_root()
3698
            and self.target.supports_rich_root()):
3699
            self._converting_to_rich_root = True
3700
            self._revision_id_to_root_id = {}
3701
        else:
3702
            self._converting_to_rich_root = False
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3703
        revision_ids = self.target.search_missing_revision_ids(self.source,
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3704
            revision_id, find_ghosts=find_ghosts).get_keys()
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3705
        if not revision_ids:
3706
            return 0, 0
3184.1.8 by Robert Collins
* ``InterRepository.missing_revision_ids`` is now deprecated in favour of
3707
        revision_ids = tsort.topo_sort(
3184.1.9 by Robert Collins
* ``Repository.get_data_stream`` is now deprecated in favour of
3708
            self.source.get_graph().get_parent_map(revision_ids))
3735.2.15 by Robert Collins
More direct implementation of fetch between different serializers.
3709
        if not revision_ids:
3710
            return 0, 0
3711
        # Walk though all revisions; get inventory deltas, copy referenced
3712
        # texts that delta references, insert the delta, revision and
3713
        # signature.
3714
        first_rev = self.source.get_revision(revision_ids[0])
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
3715
        if pb is None:
3716
            my_pb = ui.ui_factory.nested_progress_bar()
3717
            pb = my_pb
3718
        else:
4110.2.5 by Martin Pool
Deprecate passing pbs in to fetch()
3719
            symbol_versioning.warn(
3720
                symbol_versioning.deprecated_in((1, 14, 0))
3721
                % "pb parameter to fetch()")
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
3722
            my_pb = None
3723
        try:
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3724
            self._fetch_all_revisions(revision_ids, pb)
3146.6.1 by Aaron Bentley
InterDifferingSerializer shows a progress bar
3725
        finally:
3726
            if my_pb is not None:
3727
                my_pb.finished()
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3728
        return len(revision_ids), 0
3729
3879.2.8 by John Arbash Meinel
Bring in the CHK inter-differing-serializer fetch code.
3730
    def _get_basis(self, first_revision_id):
3731
        """Get a revision and tree which exists in the target.
3732
3733
        This assumes that first_revision_id is selected for transmission
3734
        because all other ancestors are already present. If we can't find an
3735
        ancestor we fall back to NULL_REVISION since we know that is safe.
3736
3737
        :return: (basis_id, basis_tree)
3738
        """
3739
        first_rev = self.source.get_revision(first_revision_id)
3740
        try:
3741
            basis_id = first_rev.parent_ids[0]
3742
            # only valid as a basis if the target has it
3743
            self.target.get_revision(basis_id)
3744
            # Try to get a basis tree - if its a ghost it will hit the
3745
            # NoSuchRevision case.
3746
            basis_tree = self.source.revision_tree(basis_id)
3747
        except (IndexError, errors.NoSuchRevision):
3748
            basis_id = _mod_revision.NULL_REVISION
3749
            basis_tree = self.source.revision_tree(basis_id)
3750
        return basis_id, basis_tree
3751
2996.2.1 by Aaron Bentley
Add KnitRepositoryFormat4
3752
3753
InterRepository.register_optimiser(InterDifferingSerializer)
1910.2.15 by Aaron Bentley
Back out inter.get changes, make optimizers an ordered list
3754
InterRepository.register_optimiser(InterSameDataRepository)
2241.1.13 by Martin Pool
Re-register InterWeaveRepo, fix test integration, add test for it
3755
InterRepository.register_optimiser(InterWeaveRepo)
1563.2.31 by Robert Collins
Convert Knit repositories to use knits.
3756
InterRepository.register_optimiser(InterKnitRepo)
2592.3.90 by Robert Collins
Slightly broken, but branch and fetch performance is now roughly on par (for bzr.dev) with knits - should be much faster for large repos.
3757
InterRepository.register_optimiser(InterPackRepo)
1534.1.31 by Robert Collins
Deprecated fetch.fetch and fetch.greedy_fetch for branch.fetch, and move the Repository.fetch internals to InterRepo and InterWeaveRepo.
3758
3759
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3760
class CopyConverter(object):
3761
    """A repository conversion tool which just performs a copy of the content.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3762
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3763
    This is slow but quite reliable.
3764
    """
3765
3766
    def __init__(self, target_format):
3767
        """Create a CopyConverter.
3768
3769
        :param target_format: The format the resulting repository should be.
3770
        """
3771
        self.target_format = target_format
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3772
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3773
    def convert(self, repo, pb):
3774
        """Perform the conversion of to_convert, giving feedback via pb.
3775
3776
        :param to_convert: The disk object to convert.
3777
        :param pb: a progress bar to use for progress information.
3778
        """
3779
        self.pb = pb
3780
        self.count = 0
1596.2.22 by Robert Collins
Fetch changes to use new pb.
3781
        self.total = 4
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3782
        # this is only useful with metadir layouts - separated repo content.
3783
        # trigger an assertion if not such
3784
        repo._format.get_format_string()
3785
        self.repo_dir = repo.bzrdir
3786
        self.step('Moving repository to repository.backup')
3787
        self.repo_dir.transport.move('repository', 'repository.backup')
3788
        backup_transport =  self.repo_dir.transport.clone('repository.backup')
1910.2.12 by Aaron Bentley
Implement knit repo format 2
3789
        repo._format.check_conversion_target(self.target_format)
1556.1.4 by Robert Collins
Add a new format for what will become knit, and the surrounding logic to upgrade repositories within metadirs, and tests for the same.
3790
        self.source_repo = repo._format.open(self.repo_dir,
3791
            _found=True,
3792
            _override_transport=backup_transport)
3793
        self.step('Creating new repository')
3794
        converted = self.target_format.initialize(self.repo_dir,
3795
                                                  self.source_repo.is_shared())
3796
        converted.lock_write()
3797
        try:
3798
            self.step('Copying content into repository.')
3799
            self.source_repo.copy_content_into(converted)
3800
        finally:
3801
            converted.unlock()
3802
        self.step('Deleting old repository content.')
3803
        self.repo_dir.transport.delete_tree('repository.backup')
3804
        self.pb.note('repository converted')
3805
3806
    def step(self, message):
3807
        """Update the pb by a step."""
3808
        self.count +=1
3809
        self.pb.update(message, self.count, self.total)
1596.1.1 by Martin Pool
Use simple xml unescaping rather than importing xml.sax
3810
3811
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3812
_unescape_map = {
3813
    'apos':"'",
3814
    'quot':'"',
3815
    'amp':'&',
3816
    'lt':'<',
3817
    'gt':'>'
3818
}
3819
3820
3821
def _unescaper(match, _map=_unescape_map):
2294.1.2 by John Arbash Meinel
Track down and add tests that all tree.commit() can handle
3822
    code = match.group(1)
3823
    try:
3824
        return _map[code]
3825
    except KeyError:
3826
        if not code.startswith('#'):
3827
            raise
2294.1.10 by John Arbash Meinel
Switch all apis over to utf8 file ids. All tests pass
3828
        return unichr(int(code[1:])).encode('utf8')
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3829
3830
3831
_unescape_re = None
3832
3833
1596.1.1 by Martin Pool
Use simple xml unescaping rather than importing xml.sax
3834
def _unescape_xml(data):
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3835
    """Unescape predefined XML entities in a string of data."""
3836
    global _unescape_re
3837
    if _unescape_re is None:
2120.2.1 by John Arbash Meinel
Remove tabs from source files, and add a test to keep it that way.
3838
        _unescape_re = re.compile('\&([^;]*);')
1843.2.4 by Aaron Bentley
Switch to John Meinel's _unescape_xml implementation
3839
    return _unescape_re.sub(_unescaper, data)
2745.6.3 by Aaron Bentley
Implement versionedfile checking for bzr check
3840
3841
3036.1.3 by Robert Collins
Privatise VersionedFileChecker.
3842
class _VersionedFileChecker(object):
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3843
4145.2.1 by Ian Clatworthy
faster check
3844
    def __init__(self, repository, text_key_references=None):
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3845
        self.repository = repository
4145.2.1 by Ian Clatworthy
faster check
3846
        self.text_index = self.repository._generate_text_key_index(
3847
            text_key_references=text_key_references)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
3848
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3849
    def calculate_file_version_parents(self, text_key):
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3850
        """Calculate the correct parents for a file version according to
3851
        the inventories.
3852
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3853
        parent_keys = self.text_index[text_key]
2988.1.8 by Robert Collins
Change check and reconcile to use the new _generate_text_key_index rather
3854
        if parent_keys == [_mod_revision.NULL_REVISION]:
3855
            return ()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3856
        return tuple(parent_keys)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
3857
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3858
    def check_file_version_parents(self, texts, progress_bar=None):
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3859
        """Check the parents stored in a versioned file are correct.
3860
3861
        It also detects file versions that are not referenced by their
3862
        corresponding revision's inventory.
3863
2927.2.14 by Andrew Bennetts
Tweaks suggested by review.
3864
        :returns: A tuple of (wrong_parents, dangling_file_versions).
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3865
            wrong_parents is a dict mapping {revision_id: (stored_parents,
3866
            correct_parents)} for each revision_id where the stored parents
2927.2.14 by Andrew Bennetts
Tweaks suggested by review.
3867
            are not correct.  dangling_file_versions is a set of (file_id,
3868
            revision_id) tuples for versions that are present in this versioned
3869
            file, but not used by the corresponding inventory.
2927.2.10 by Andrew Bennetts
More docstrings, elaborate a comment with an XXX, and remove a little bit of cruft.
3870
        """
2927.2.3 by Andrew Bennetts
Add fulltexts to avoid bug 155730.
3871
        wrong_parents = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3872
        self.file_ids = set([file_id for file_id, _ in
3873
            self.text_index.iterkeys()])
3874
        # text keys is now grouped by file_id
3875
        n_weaves = len(self.file_ids)
3876
        files_in_revisions = {}
3877
        revisions_of_files = {}
3878
        n_versions = len(self.text_index)
3879
        progress_bar.update('loading text store', 0, n_versions)
3880
        parent_map = self.repository.texts.get_parent_map(self.text_index)
3881
        # On unlistable transports this could well be empty/error...
3882
        text_keys = self.repository.texts.keys()
3883
        unused_keys = frozenset(text_keys) - set(self.text_index)
3884
        for num, key in enumerate(self.text_index.iterkeys()):
3885
            if progress_bar is not None:
3886
                progress_bar.update('checking text graph', num, n_versions)
3887
            correct_parents = self.calculate_file_version_parents(key)
2927.2.6 by Andrew Bennetts
Make some more check tests pass.
3888
            try:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
3889
                knit_parents = parent_map[key]
3890
            except errors.RevisionNotPresent:
3891
                # Missing text!
3892
                knit_parents = None
3893
            if correct_parents != knit_parents:
3894
                wrong_parents[key] = (knit_parents, correct_parents)
3895
        return wrong_parents, unused_keys
3287.6.8 by Robert Collins
Reduce code duplication as per review.
3896
3897
3898
def _old_get_graph(repository, revision_id):
3899
    """DO NOT USE. That is all. I'm serious."""
3900
    graph = repository.get_graph()
3901
    revision_graph = dict(((key, value) for key, value in
3902
        graph.iter_ancestry([revision_id]) if value is not None))
3903
    return _strip_NULL_ghosts(revision_graph)
3904
3905
3906
def _strip_NULL_ghosts(revision_graph):
3907
    """Also don't use this. more compatibility code for unmigrated clients."""
3908
    # Filter ghosts, and null:
3909
    if _mod_revision.NULL_REVISION in revision_graph:
3910
        del revision_graph[_mod_revision.NULL_REVISION]
3911
    for key, parents in revision_graph.items():
3912
        revision_graph[key] = tuple(parent for parent in parents if parent
3913
            in revision_graph)
3914
    return revision_graph
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3915
3916
3917
class StreamSink(object):
3918
    """An object that can insert a stream into a repository.
3919
3920
    This interface handles the complexity of reserialising inventories and
3921
    revisions from different formats, and allows unidirectional insertion into
3922
    stacked repositories without looking for the missing basis parents
3923
    beforehand.
3924
    """
3925
3926
    def __init__(self, target_repo):
3927
        self.target_repo = target_repo
3928
4032.3.7 by Robert Collins
Move write locking and write group responsibilities into the Sink objects themselves, allowing complete avoidance of unnecessary calls when the sink is a RemoteSink.
3929
    def insert_stream(self, stream, src_format, resume_tokens):
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3930
        """Insert a stream's content into the target repository.
3931
3932
        :param src_format: a bzr repository format.
3933
4032.3.7 by Robert Collins
Move write locking and write group responsibilities into the Sink objects themselves, allowing complete avoidance of unnecessary calls when the sink is a RemoteSink.
3934
        :return: a list of resume tokens and an  iterable of keys additional
3935
            items required before the insertion can be completed.
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3936
        """
4032.3.7 by Robert Collins
Move write locking and write group responsibilities into the Sink objects themselves, allowing complete avoidance of unnecessary calls when the sink is a RemoteSink.
3937
        self.target_repo.lock_write()
3938
        try:
3939
            if resume_tokens:
3940
                self.target_repo.resume_write_group(resume_tokens)
3941
            else:
3942
                self.target_repo.start_write_group()
3943
            try:
3944
                # locked_insert_stream performs a commit|suspend.
3945
                return self._locked_insert_stream(stream, src_format)
3946
            except:
3947
                self.target_repo.abort_write_group(suppress_errors=True)
3948
                raise
3949
        finally:
3950
            self.target_repo.unlock()
3951
3952
    def _locked_insert_stream(self, stream, src_format):
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3953
        to_serializer = self.target_repo._format._serializer
3954
        src_serializer = src_format._serializer
4187.3.2 by Andrew Bennetts
Only enable the hack when the serializers match, otherwise we cause ShortReadvErrors.
3955
        if to_serializer == src_serializer:
3956
            # If serializers match and the target is a pack repository, set the
3957
            # write cache size on the new pack.  This avoids poor performance
3958
            # on transports where append is unbuffered (such as
4187.3.4 by Andrew Bennetts
Better docstrings and comments.
3959
            # RemoteTransport).  This is safe to do because nothing should read
4187.3.2 by Andrew Bennetts
Only enable the hack when the serializers match, otherwise we cause ShortReadvErrors.
3960
            # back from the target repository while a stream with matching
3961
            # serialization is being inserted.
4187.3.4 by Andrew Bennetts
Better docstrings and comments.
3962
            # The exception is that a delta record from the source that should
3963
            # be a fulltext may need to be expanded by the target (see
3964
            # test_fetch_revisions_with_deltas_into_pack); but we take care to
3965
            # explicitly flush any buffered writes first in that rare case.
4187.3.2 by Andrew Bennetts
Only enable the hack when the serializers match, otherwise we cause ShortReadvErrors.
3966
            try:
3967
                new_pack = self.target_repo._pack_collection._new_pack
3968
            except AttributeError:
3969
                # Not a pack repository
3970
                pass
3971
            else:
3972
                new_pack.set_write_cache_size(1024*1024)
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3973
        for substream_type, substream in stream:
3974
            if substream_type == 'texts':
3975
                self.target_repo.texts.insert_record_stream(substream)
3976
            elif substream_type == 'inventories':
3977
                if src_serializer == to_serializer:
3978
                    self.target_repo.inventories.insert_record_stream(
3979
                        substream)
3980
                else:
3981
                    self._extract_and_insert_inventories(
3982
                        substream, src_serializer)
3735.2.98 by John Arbash Meinel
Merge bzr.dev 4032. Resolve the new streaming fetch.
3983
            elif substream_type == 'chk_bytes':
3984
                # XXX: This doesn't support conversions, as it assumes the
3985
                #      conversion was done in the fetch code.
3986
                self.target_repo.chk_bytes.insert_record_stream(substream)
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
3987
            elif substream_type == 'revisions':
3988
                # This may fallback to extract-and-insert more often than
3989
                # required if the serializers are different only in terms of
3990
                # the inventory.
3991
                if src_serializer == to_serializer:
3992
                    self.target_repo.revisions.insert_record_stream(
3993
                        substream)
3994
                else:
3995
                    self._extract_and_insert_revisions(substream,
3996
                        src_serializer)
3997
            elif substream_type == 'signatures':
3998
                self.target_repo.signatures.insert_record_stream(substream)
3999
            else:
4000
                raise AssertionError('kaboom! %s' % (substream_type,))
4032.3.7 by Robert Collins
Move write locking and write group responsibilities into the Sink objects themselves, allowing complete avoidance of unnecessary calls when the sink is a RemoteSink.
4001
        try:
4002
            missing_keys = set()
4003
            for prefix, versioned_file in (
4004
                ('texts', self.target_repo.texts),
4005
                ('inventories', self.target_repo.inventories),
4006
                ('revisions', self.target_repo.revisions),
4007
                ('signatures', self.target_repo.signatures),
4008
                ):
4009
                missing_keys.update((prefix,) + key for key in
4010
                    versioned_file.get_missing_compression_parent_keys())
4011
        except NotImplementedError:
4012
            # cannot even attempt suspending, and missing would have failed
4013
            # during stream insertion.
4014
            missing_keys = set()
4015
        else:
4016
            if missing_keys:
4017
                # suspend the write group and tell the caller what we is
4018
                # missing. We know we can suspend or else we would not have
4019
                # entered this code path. (All repositories that can handle
4020
                # missing keys can handle suspending a write group).
4021
                write_group_tokens = self.target_repo.suspend_write_group()
4022
                return write_group_tokens, missing_keys
4023
        self.target_repo.commit_write_group()
4024
        return [], set()
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
4025
4026
    def _extract_and_insert_inventories(self, substream, serializer):
4027
        """Generate a new inventory versionedfile in target, converting data.
4032.1.1 by John Arbash Meinel
Merge the removal of all trailing whitespace, and resolve conflicts.
4028
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
4029
        The inventory is retrieved from the source, (deserializing it), and
4030
        stored in the target (reserializing it in a different format).
4031
        """
4032
        for record in substream:
4033
            bytes = record.get_bytes_as('fulltext')
4034
            revision_id = record.key[0]
4035
            inv = serializer.read_inventory_from_string(bytes, revision_id)
4036
            parents = [key[0] for key in record.parents]
4037
            self.target_repo.add_inventory(revision_id, inv, parents)
4038
4039
    def _extract_and_insert_revisions(self, substream, serializer):
4040
        for record in substream:
4041
            bytes = record.get_bytes_as('fulltext')
4042
            revision_id = record.key[0]
4043
            rev = serializer.read_revision_from_string(bytes)
4044
            if rev.revision_id != revision_id:
4045
                raise AssertionError('wtf: %s != %s' % (rev, revision_id))
4046
            self.target_repo.add_revision(revision_id, rev)
4047
4048
    def finished(self):
4053.1.4 by Robert Collins
Move the fetch control attributes from Repository to RepositoryFormat.
4049
        if self.target_repo._format._fetch_reconcile:
4022.1.1 by Robert Collins
Refactoring of fetch to have a sender and sink component enabling splitting the logic over a network stream. (Robert Collins, Andrew Bennetts)
4050
            self.target_repo.reconcile()
4051
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
4052
4053
class StreamSource(object):
4065.1.2 by Robert Collins
Merge bzr.dev [fix conflicts with fetch refactoring].
4054
    """A source of a stream for fetching between repositories."""
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
4055
4056
    def __init__(self, from_repository, to_format):
4057
        """Create a StreamSource streaming from from_repository."""
4058
        self.from_repository = from_repository
4059
        self.to_format = to_format
4060
4061
    def delta_on_metadata(self):
4062
        """Return True if delta's are permitted on metadata streams.
4063
4064
        That is on revisions and signatures.
4065
        """
4066
        src_serializer = self.from_repository._format._serializer
4067
        target_serializer = self.to_format._serializer
4068
        return (self.to_format._fetch_uses_deltas and
4069
            src_serializer == target_serializer)
4070
4071
    def _fetch_revision_texts(self, revs):
4072
        # fetch signatures first and then the revision texts
4073
        # may need to be a InterRevisionStore call here.
4074
        from_sf = self.from_repository.signatures
4075
        # A missing signature is just skipped.
4076
        keys = [(rev_id,) for rev_id in revs]
4060.1.4 by Robert Collins
Streaming fetch from remote servers.
4077
        signatures = versionedfile.filter_absent(from_sf.get_record_stream(
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
4078
            keys,
4079
            self.to_format._fetch_order,
4080
            not self.to_format._fetch_uses_deltas))
4081
        # If a revision has a delta, this is actually expanded inside the
4082
        # insert_record_stream code now, which is an alternate fix for
4083
        # bug #261339
4084
        from_rf = self.from_repository.revisions
4085
        revisions = from_rf.get_record_stream(
4086
            keys,
4087
            self.to_format._fetch_order,
4088
            not self.delta_on_metadata())
4089
        return [('signatures', signatures), ('revisions', revisions)]
4090
4091
    def _generate_root_texts(self, revs):
4092
        """This will be called by __fetch between fetching weave texts and
4093
        fetching the inventory weave.
4094
4095
        Subclasses should override this if they need to generate root texts
4096
        after fetching weave texts.
4097
        """
4098
        if self._rich_root_upgrade():
4099
            import bzrlib.fetch
4100
            return bzrlib.fetch.Inter1and2Helper(
4101
                self.from_repository).generate_root_texts(revs)
4102
        else:
4103
            return []
4104
4105
    def get_stream(self, search):
4106
        phase = 'file'
4107
        revs = search.get_keys()
4108
        graph = self.from_repository.get_graph()
4109
        revs = list(graph.iter_topo_order(revs))
4110
        data_to_fetch = self.from_repository.item_keys_introduced_by(revs)
4111
        text_keys = []
4112
        for knit_kind, file_id, revisions in data_to_fetch:
4113
            if knit_kind != phase:
4114
                phase = knit_kind
4115
                # Make a new progress bar for this phase
4116
            if knit_kind == "file":
4117
                # Accumulate file texts
4118
                text_keys.extend([(file_id, revision) for revision in
4119
                    revisions])
4120
            elif knit_kind == "inventory":
4121
                # Now copy the file texts.
4122
                from_texts = self.from_repository.texts
4123
                yield ('texts', from_texts.get_record_stream(
4124
                    text_keys, self.to_format._fetch_order,
4125
                    not self.to_format._fetch_uses_deltas))
4126
                # Cause an error if a text occurs after we have done the
4127
                # copy.
4128
                text_keys = None
4129
                # Before we process the inventory we generate the root
4130
                # texts (if necessary) so that the inventories references
4131
                # will be valid.
4132
                for _ in self._generate_root_texts(revs):
4133
                    yield _
4134
                # NB: This currently reopens the inventory weave in source;
4135
                # using a single stream interface instead would avoid this.
4136
                from_weave = self.from_repository.inventories
4137
                # we fetch only the referenced inventories because we do not
4138
                # know for unselected inventories whether all their required
4139
                # texts are present in the other repository - it could be
4140
                # corrupt.
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
4141
                for info in self._get_inventory_stream(revs):
4142
                    yield info
4060.1.3 by Robert Collins
Implement the separate source component for fetch - repository.StreamSource.
4143
            elif knit_kind == "signatures":
4144
                # Nothing to do here; this will be taken care of when
4145
                # _fetch_revision_texts happens.
4146
                pass
4147
            elif knit_kind == "revisions":
4148
                for record in self._fetch_revision_texts(revs):
4149
                    yield record
4150
            else:
4151
                raise AssertionError("Unknown knit kind %r" % knit_kind)
4152
4153
    def get_stream_for_missing_keys(self, missing_keys):
4154
        # missing keys can only occur when we are byte copying and not
4155
        # translating (because translation means we don't send
4156
        # unreconstructable deltas ever).
4157
        keys = {}
4158
        keys['texts'] = set()
4159
        keys['revisions'] = set()
4160
        keys['inventories'] = set()
4161
        keys['signatures'] = set()
4162
        for key in missing_keys:
4163
            keys[key[0]].add(key[1:])
4164
        if len(keys['revisions']):
4165
            # If we allowed copying revisions at this point, we could end up
4166
            # copying a revision without copying its required texts: a
4167
            # violation of the requirements for repository integrity.
4168
            raise AssertionError(
4169
                'cannot copy revisions to fill in missing deltas %s' % (
4170
                    keys['revisions'],))
4171
        for substream_kind, keys in keys.iteritems():
4172
            vf = getattr(self.from_repository, substream_kind)
4173
            # Ask for full texts always so that we don't need more round trips
4174
            # after this stream.
4175
            stream = vf.get_record_stream(keys,
4176
                self.to_format._fetch_order, True)
4177
            yield substream_kind, stream
4178
4179
    def inventory_fetch_order(self):
4180
        if self._rich_root_upgrade():
4181
            return 'topological'
4182
        else:
4183
            return self.to_format._fetch_order
4184
4185
    def _rich_root_upgrade(self):
4186
        return (not self.from_repository._format.rich_root_data and
4187
            self.to_format.rich_root_data)
4188
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
4189
    def _get_inventory_stream(self, revision_ids):
4190
        from_format = self.from_repository._format
4191
        if (from_format.supports_chks and self.to_format.supports_chks
4192
            and (from_format._serializer == self.to_format._serializer)):
4193
            # Both sides support chks, and they use the same serializer, so it
4194
            # is safe to transmit the chk pages and inventory pages across
4195
            # as-is.
4196
            return self._get_chk_inventory_stream(revision_ids)
4197
        elif (not from_format.supports_chks):
4198
            # Source repository doesn't support chks. So we can transmit the
4199
            # inventories 'as-is' and either they are just accepted on the
4200
            # target, or the Sink will properly convert it.
4201
            return self._get_simple_inventory_stream(revision_ids)
4202
        else:
4203
            # XXX: Hack to make not-chk->chk fetch: copy the inventories as
4204
            #      inventories. Note that this should probably be done somehow
4205
            #      as part of bzrlib.repository.StreamSink. Except JAM couldn't
4206
            #      figure out how a non-chk repository could possibly handle
4207
            #      deserializing an inventory stream from a chk repo, as it
4208
            #      doesn't have a way to understand individual pages.
4209
            return self._get_convertable_inventory_stream(revision_ids)
4210
4211
    def _get_simple_inventory_stream(self, revision_ids):
4212
        from_weave = self.from_repository.inventories
4213
        yield ('inventories', from_weave.get_record_stream(
4214
            [(rev_id,) for rev_id in revision_ids],
4215
            self.inventory_fetch_order(),
4216
            not self.delta_on_metadata()))
4217
4218
    def _get_chk_inventory_stream(self, revision_ids):
4219
        """Fetch the inventory texts, along with the associated chk maps."""
4220
        # We want an inventory outside of the search set, so that we can filter
4221
        # out uninteresting chk pages. For now we use
4222
        # _find_revision_outside_set, but if we had a Search with cut_revs, we
4223
        # could use that instead.
4224
        start_rev_id = self.from_repository._find_revision_outside_set(
4225
                            revision_ids)
4226
        start_rev_key = (start_rev_id,)
4227
        inv_keys_to_fetch = [(rev_id,) for rev_id in revision_ids]
4228
        if start_rev_id != _mod_revision.NULL_REVISION:
4229
            inv_keys_to_fetch.append((start_rev_id,))
4230
        # Any repo that supports chk_bytes must also support out-of-order
4231
        # insertion. At least, that is how we expect it to work
4232
        # We use get_record_stream instead of iter_inventories because we want
4233
        # to be able to insert the stream as well. We could instead fetch
4234
        # allowing deltas, and then iter_inventories, but we don't know whether
4235
        # source or target is more 'local' anway.
4236
        inv_stream = self.from_repository.inventories.get_record_stream(
4237
            inv_keys_to_fetch, 'unordered',
4238
            True) # We need them as full-texts so we can find their references
4239
        uninteresting_chk_roots = set()
4240
        interesting_chk_roots = set()
4241
        def filter_inv_stream(inv_stream):
4242
            for idx, record in enumerate(inv_stream):
4243
                ### child_pb.update('fetch inv', idx, len(inv_keys_to_fetch))
4244
                bytes = record.get_bytes_as('fulltext')
4245
                chk_inv = inventory.CHKInventory.deserialise(
4246
                    self.from_repository.chk_bytes, bytes, record.key)
4247
                if record.key == start_rev_key:
4248
                    uninteresting_chk_roots.add(chk_inv.id_to_entry.key())
4249
                    p_id_map = chk_inv.parent_id_basename_to_file_id
4250
                    if p_id_map is not None:
4251
                        uninteresting_chk_roots.add(p_id_map.key())
4252
                else:
4253
                    yield record
4254
                    interesting_chk_roots.add(chk_inv.id_to_entry.key())
4255
                    p_id_map = chk_inv.parent_id_basename_to_file_id
4256
                    if p_id_map is not None:
4257
                        interesting_chk_roots.add(p_id_map.key())
4258
        ### pb.update('fetch inventory', 0, 2)
4259
        yield ('inventories', filter_inv_stream(inv_stream))
4260
        # Now that we have worked out all of the interesting root nodes, grab
4261
        # all of the interesting pages and insert them
4262
        ### pb.update('fetch inventory', 1, 2)
4263
        interesting = chk_map.iter_interesting_nodes(
4264
            self.from_repository.chk_bytes, interesting_chk_roots,
4265
            uninteresting_chk_roots)
4266
        def to_stream_adapter():
4267
            """Adapt the iter_interesting_nodes result to a single stream.
4268
3735.37.4 by John Arbash Meinel
Change the iter_interesting_nodes api, it can now explicitly only return
4269
            iter_interesting_nodes returns records as it processes them, along
4270
            with keys. However, we only want to return the records themselves.
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
4271
            """
4272
            for record, items in interesting:
3735.37.4 by John Arbash Meinel
Change the iter_interesting_nodes api, it can now explicitly only return
4273
                if record is not None:
4274
                    yield record
3735.2.128 by Andrew Bennetts
Merge bzr.dev, resolving fetch.py conflict.
4275
        # XXX: We could instead call get_record_stream(records.keys())
4276
        #      ATM, this will always insert the records as fulltexts, and
4277
        #      requires that you can hang on to records once you have gone
4278
        #      on to the next one. Further, it causes the target to
4279
        #      recompress the data. Testing shows it to be faster than
4280
        #      requesting the records again, though.
4281
        yield ('chk_bytes', to_stream_adapter())
4282
        ### pb.update('fetch inventory', 2, 2)
4283
4284
    def _get_convertable_inventory_stream(self, revision_ids):
4285
        # XXX: One of source or target is using chks, and they don't have
4286
        #      compatible serializations. The StreamSink code expects to be
4287
        #      able to convert on the target, so we need to put
4288
        #      bytes-on-the-wire that can be converted
4289
        yield ('inventories', self._stream_invs_as_fulltexts(revision_ids))
4290
4291
    def _stream_invs_as_fulltexts(self, revision_ids):
4292
        from_repo = self.from_repository
4293
        from_serializer = from_repo._format._serializer
4294
        revision_keys = [(rev_id,) for rev_id in revision_ids]
4295
        parent_map = from_repo.inventories.get_parent_map(revision_keys)
4296
        for inv in self.from_repository.iter_inventories(revision_ids):
4297
            # XXX: This is a bit hackish, but it works. Basically,
4298
            #      CHKSerializer 'accidentally' supports
4299
            #      read/write_inventory_to_string, even though that is never
4300
            #      the format that is stored on disk. It *does* give us a
4301
            #      single string representation for an inventory, so live with
4302
            #      it for now.
4303
            #      This would be far better if we had a 'serialized inventory
4304
            #      delta' form. Then we could use 'inventory._make_delta', and
4305
            #      transmit that. This would both be faster to generate, and
4306
            #      result in fewer bytes-on-the-wire.
4307
            as_bytes = from_serializer.write_inventory_to_string(inv)
4308
            key = (inv.revision_id,)
4309
            parent_keys = parent_map.get(key, ())
4310
            yield versionedfile.FulltextContentFactory(
4311
                key, parent_keys, None, as_bytes)
4312