1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
"""Copying of history from one branch to another.
20
The basic plan is that every branch knows the history of everything
21
that has merged into it. As the first step of a merge, pull, or
22
branch operation we copy history from the source into the destination
25
The copying is done in a slightly complicated order. We don't want to
26
add a revision to the store until everything it refers to is also
27
stored, so that if a revision is present we can totally recreate it.
28
However, we can't know what files are included in a revision until we
29
read its inventory. So we query the inventory store of the source for
30
the ids we need, and then pull those ids and finally actually join
35
import bzrlib.errors as errors
36
from bzrlib.errors import (InstallFailed,
38
from bzrlib.progress import ProgressPhase
39
from bzrlib.revision import NULL_REVISION
40
from bzrlib.symbol_versioning import (deprecated_function,
43
from bzrlib.trace import mutter
46
from bzrlib.lazy_import import lazy_import
48
# TODO: Avoid repeatedly opening weaves so many times.
50
# XXX: This doesn't handle ghost (not present in branch) revisions at
51
# all yet. I'm not sure they really should be supported.
53
# NOTE: This doesn't copy revisions which may be present but not
54
# merged into the last revision. I'm not sure we want to do that.
56
# - get a list of revisions that need to be pulled in
57
# - for each one, pull in that revision file
58
# and get the inventory, and store the inventory with right
60
# - and get the ancestry, and store that with right parents too
61
# - and keep a note of all file ids and version seen
62
# - then go through all files; for each one get the weave,
63
# and add in all file versions
66
class RepoFetcher(object):
67
"""Pull revisions and texts from one repository to another.
70
if set, try to limit to the data this revision references.
73
count_copied -- number of revisions copied
75
This should not be used directory, its essential a object to encapsulate
76
the logic in InterRepository.fetch().
78
def __init__(self, to_repository, from_repository, last_revision=None, pb=None):
80
self.failed_revisions = []
82
if to_repository.control_files._transport.base == from_repository.control_files._transport.base:
83
# check that last_revision is in 'from' and then return a no-operation.
84
if last_revision not in (None, NULL_REVISION):
85
to_repository.get_revision(last_revision)
87
self.to_repository = to_repository
88
self.from_repository = from_repository
89
# must not mutate self._last_revision as its potentially a shared instance
90
self._last_revision = last_revision
92
self.pb = bzrlib.ui.ui_factory.nested_progress_bar()
93
self.nested_pb = self.pb
97
self.from_repository.lock_read()
99
self.to_repository.lock_write()
101
self.to_repository.start_write_group()
105
self.to_repository.abort_write_group()
108
self.to_repository.commit_write_group()
110
if self.nested_pb is not None:
111
self.nested_pb.finished()
112
self.to_repository.unlock()
114
self.from_repository.unlock()
117
"""Primary worker function.
119
This initialises all the needed variables, and then fetches the
120
requested revisions, finally clearing the progress bar.
122
self.to_weaves = self.to_repository.weave_store
123
self.to_control = self.to_repository.control_weaves
124
self.from_weaves = self.from_repository.weave_store
125
self.from_control = self.from_repository.control_weaves
127
self.file_ids_names = {}
128
pp = ProgressPhase('Fetch phase', 4, self.pb)
131
revs = self._revids_to_fetch()
135
self._fetch_weave_texts(revs)
137
self._fetch_inventory_weave(revs)
139
self._fetch_revision_texts(revs)
140
self.count_copied += len(revs)
144
def _revids_to_fetch(self):
145
mutter('fetch up to rev {%s}', self._last_revision)
146
if self._last_revision is NULL_REVISION:
147
# explicit limit of no revisions needed
149
if (self._last_revision is not None and
150
self.to_repository.has_revision(self._last_revision)):
154
return self.to_repository.missing_revision_ids(self.from_repository,
156
except errors.NoSuchRevision:
157
raise InstallFailed([self._last_revision])
159
def _fetch_weave_texts(self, revs):
160
texts_pb = bzrlib.ui.ui_factory.nested_progress_bar()
162
# fileids_altered_by_revision_ids requires reading the inventory
163
# weave, we will need to read the inventory weave again when
164
# all this is done, so enable caching for that specific weave
165
inv_w = self.from_repository.get_inventory_weave()
167
file_ids = self.from_repository.fileids_altered_by_revision_ids(revs)
169
num_file_ids = len(file_ids)
170
for file_id, required_versions in file_ids.items():
171
texts_pb.update("fetch texts", count, num_file_ids)
173
to_weave = self.to_weaves.get_weave_or_empty(file_id,
174
self.to_repository.get_transaction())
175
from_weave = self.from_weaves.get_weave(file_id,
176
self.from_repository.get_transaction())
177
# we fetch all the texts, because texts do
178
# not reference anything, and its cheap enough
179
to_weave.join(from_weave, version_ids=required_versions)
180
# we don't need *all* of this data anymore, but we dont know
181
# what we do. This cache clearing will result in a new read
182
# of the knit data when we do the checkout, but probably we
183
# want to emit the needed data on the fly rather than at the
185
# the from weave should know not to cache data being joined,
186
# but its ok to ask it to clear.
187
from_weave.clear_cache()
188
to_weave.clear_cache()
192
def _fetch_inventory_weave(self, revs):
193
pb = bzrlib.ui.ui_factory.nested_progress_bar()
195
pb.update("fetch inventory", 0, 2)
196
to_weave = self.to_control.get_weave('inventory',
197
self.to_repository.get_transaction())
199
child_pb = bzrlib.ui.ui_factory.nested_progress_bar()
201
# just merge, this is optimisable and its means we don't
202
# copy unreferenced data such as not-needed inventories.
203
pb.update("fetch inventory", 1, 3)
204
from_weave = self.from_repository.get_inventory_weave()
205
pb.update("fetch inventory", 2, 3)
206
# we fetch only the referenced inventories because we do not
207
# know for unselected inventories whether all their required
208
# texts are present in the other repository - it could be
210
to_weave.join(from_weave, pb=child_pb, msg='merge inventory',
212
from_weave.clear_cache()
219
class GenericRepoFetcher(RepoFetcher):
220
"""This is a generic repo to repo fetcher.
222
This makes minimal assumptions about repo layout and contents.
223
It triggers a reconciliation after fetching to ensure integrity.
226
def _fetch_revision_texts(self, revs):
227
"""Fetch revision object texts"""
228
rev_pb = bzrlib.ui.ui_factory.nested_progress_bar()
230
to_txn = self.to_transaction = self.to_repository.get_transaction()
233
to_store = self.to_repository._revision_store
235
pb = bzrlib.ui.ui_factory.nested_progress_bar()
237
pb.update('copying revisions', count, total)
239
sig_text = self.from_repository.get_signature_text(rev)
240
to_store.add_revision_signature_text(rev, sig_text, to_txn)
241
except errors.NoSuchRevision:
244
to_store.add_revision(self.from_repository.get_revision(rev),
249
# fixup inventory if needed:
250
# this is expensive because we have no inverse index to current ghosts.
251
# but on local disk its a few seconds and sftp push is already insane.
253
# FIXME: repository should inform if this is needed.
254
self.to_repository.reconcile()
259
class KnitRepoFetcher(RepoFetcher):
260
"""This is a knit format repository specific fetcher.
262
This differs from the GenericRepoFetcher by not doing a
263
reconciliation after copying, and using knit joining to
267
def _fetch_revision_texts(self, revs):
268
# may need to be a InterRevisionStore call here.
269
from_transaction = self.from_repository.get_transaction()
270
to_transaction = self.to_repository.get_transaction()
271
to_sf = self.to_repository._revision_store.get_signature_file(
273
from_sf = self.from_repository._revision_store.get_signature_file(
275
to_sf.join(from_sf, version_ids=revs, ignore_missing=True)
276
to_rf = self.to_repository._revision_store.get_revision_file(
278
from_rf = self.from_repository._revision_store.get_revision_file(
280
to_rf.join(from_rf, version_ids=revs)
283
class Inter1and2Helper(object):
284
"""Helper for operations that convert data from model 1 and 2
286
This is for use by fetchers and converters.
289
def __init__(self, source, target):
292
:param source: The repository data comes from
293
:param target: The repository data goes to
298
def iter_rev_trees(self, revs):
299
"""Iterate through RevisionTrees efficiently.
301
Additionally, the inventory's revision_id is set if unset.
303
Trees are retrieved in batches of 100, and then yielded in the order
306
:param revs: A list of revision ids
309
for tree in self.source.revision_trees(revs[:100]):
310
if tree.inventory.revision_id is None:
311
tree.inventory.revision_id = tree.get_revision_id()
315
def generate_root_texts(self, revs):
316
"""Generate VersionedFiles for all root ids.
318
:param revs: the revisions to include
320
inventory_weave = self.source.get_inventory_weave()
323
to_store = self.target.weave_store
324
for tree in self.iter_rev_trees(revs):
325
revision_id = tree.inventory.root.revision
326
root_id = tree.inventory.root.file_id
327
parents = inventory_weave.get_parents(revision_id)
328
if root_id not in versionedfile:
329
versionedfile[root_id] = to_store.get_weave_or_empty(root_id,
330
self.target.get_transaction())
331
parent_texts[root_id] = versionedfile[root_id].add_lines(
332
revision_id, parents, [], parent_texts)
334
def regenerate_inventory(self, revs):
335
"""Generate a new inventory versionedfile in target, convertin data.
337
The inventory is retrieved from the source, (deserializing it), and
338
stored in the target (reserializing it in a different format).
339
:param revs: The revisions to include
341
inventory_weave = self.source.get_inventory_weave()
342
for tree in self.iter_rev_trees(revs):
343
parents = inventory_weave.get_parents(tree.get_revision_id())
344
self.target.add_inventory(tree.get_revision_id(), tree.inventory,
348
class Model1toKnit2Fetcher(GenericRepoFetcher):
349
"""Fetch from a Model1 repository into a Knit2 repository
351
def __init__(self, to_repository, from_repository, last_revision=None,
353
self.helper = Inter1and2Helper(from_repository, to_repository)
354
GenericRepoFetcher.__init__(self, to_repository, from_repository,
357
def _fetch_weave_texts(self, revs):
358
GenericRepoFetcher._fetch_weave_texts(self, revs)
359
# Now generate a weave for the tree root
360
self.helper.generate_root_texts(revs)
362
def _fetch_inventory_weave(self, revs):
363
self.helper.regenerate_inventory(revs)
366
class Knit1to2Fetcher(KnitRepoFetcher):
367
"""Fetch from a Knit1 repository into a Knit2 repository"""
369
def __init__(self, to_repository, from_repository, last_revision=None,
371
self.helper = Inter1and2Helper(from_repository, to_repository)
372
KnitRepoFetcher.__init__(self, to_repository, from_repository,
375
def _fetch_weave_texts(self, revs):
376
KnitRepoFetcher._fetch_weave_texts(self, revs)
377
# Now generate a weave for the tree root
378
self.helper.generate_root_texts(revs)
380
def _fetch_inventory_weave(self, revs):
381
self.helper.regenerate_inventory(revs)
added
removed
bzrlib/fetch.py
