1
# Copyright (C) 2005, 2006 by Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
35
35
import bzrlib.errors as errors
36
from bzrlib.errors import (InstallFailed, NoSuchRevision,
36
from bzrlib.errors import (InstallFailed,
38
from bzrlib.progress import ProgressPhase
39
from bzrlib.revision import is_null, NULL_REVISION
40
from bzrlib.symbol_versioning import (deprecated_function,
38
43
from bzrlib.trace import mutter
39
from bzrlib.progress import ProgressBar, ProgressPhase
40
from bzrlib.reconcile import RepoReconciler
41
from bzrlib.revision import NULL_REVISION
42
from bzrlib.symbol_versioning import *
46
from bzrlib.lazy_import import lazy_import
45
48
# TODO: Avoid repeatedly opening weaves so many times.
119
121
requested revisions, finally clearing the progress bar.
121
123
self.to_weaves = self.to_repository.weave_store
122
self.to_control = self.to_repository.control_weaves
123
124
self.from_weaves = self.from_repository.weave_store
124
self.from_control = self.from_repository.control_weaves
125
125
self.count_total = 0
126
126
self.file_ids_names = {}
127
pp = ProgressPhase('fetch phase', 4, self.pb)
127
pp = ProgressPhase('Fetch phase', 4, self.pb)
129
130
revs = self._revids_to_fetch()
133
self._fetch_weave_texts(revs)
135
self._fetch_inventory_weave(revs)
137
self._fetch_revision_texts(revs)
138
self.count_copied += len(revs)
131
self._fetch_everything_for_revisions(revs, pp)
135
def _fetch_everything_for_revisions(self, revs, pp):
136
"""Fetch all data for the given set of revisions."""
139
# The first phase is "file". We pass the progress bar for it directly
140
# into item_keys_introduced_by, which has more information about how
141
# that phase is progressing than we do. Progress updates for the other
142
# phases are taken care of in this function.
143
# XXX: there should be a clear owner of the progress reporting. Perhaps
144
# item_keys_introduced_by should have a richer API than it does at the
145
# moment, so that it can feed the progress information back to this
148
pb = bzrlib.ui.ui_factory.nested_progress_bar()
150
data_to_fetch = self.from_repository.item_keys_introduced_by(revs, pb)
151
for knit_kind, file_id, revisions in data_to_fetch:
152
if knit_kind != phase:
154
# Make a new progress bar for this phase
157
pb = bzrlib.ui.ui_factory.nested_progress_bar()
158
if knit_kind == "file":
159
self._fetch_weave_text(file_id, revisions)
160
elif knit_kind == "inventory":
162
# Once we've processed all the files, then we generate the root
163
# texts (if necessary), then we process the inventory. It's a
164
# bit distasteful to have knit_kind == "inventory" mean this,
165
# perhaps it should happen on the first non-"file" knit, in case
166
# it's not always inventory?
167
self._generate_root_texts(revs)
168
self._fetch_inventory_weave(revs, pb)
169
elif knit_kind == "signatures":
170
# Nothing to do here; this will be taken care of when
171
# _fetch_revision_texts happens.
173
elif knit_kind == "revisions":
174
self._fetch_revision_texts(revs, pb)
176
raise AssertionError("Unknown knit kind %r" % knit_kind)
180
self.count_copied += len(revs)
142
182
def _revids_to_fetch(self):
183
"""Determines the exact revisions needed from self.from_repository to
184
install self._last_revision in self.to_repository.
186
If no revisions need to be fetched, then this just returns None.
143
188
mutter('fetch up to rev {%s}', self._last_revision)
144
189
if self._last_revision is NULL_REVISION:
145
190
# explicit limit of no revisions needed
147
if (self._last_revision != None and
192
if (self._last_revision is not None and
148
193
self.to_repository.has_revision(self._last_revision)):
154
199
except errors.NoSuchRevision:
155
200
raise InstallFailed([self._last_revision])
157
def _fetch_weave_texts(self, revs):
158
texts_pb = bzrlib.ui.ui_factory.nested_progress_bar()
160
file_ids = self.from_repository.fileid_involved_by_set(revs)
162
num_file_ids = len(file_ids)
163
for file_id in file_ids:
164
texts_pb.update("fetch texts", count, num_file_ids)
167
to_weave = self.to_weaves.get_weave(file_id,
168
self.to_repository.get_transaction())
169
except errors.NoSuchFile:
170
# destination is empty, just copy it.
171
# this copies all the texts, which is useful and
172
# on per-file basis quite cheap.
173
self.to_weaves.copy_multi(
177
self.from_repository.get_transaction(),
178
self.to_repository.get_transaction())
180
# destination has contents, must merge
181
from_weave = self.from_weaves.get_weave(file_id,
182
self.from_repository.get_transaction())
183
# we fetch all the texts, because texts do
184
# not reference anything, and its cheap enough
185
to_weave.join(from_weave)
189
def _fetch_inventory_weave(self, revs):
190
pb = bzrlib.ui.ui_factory.nested_progress_bar()
192
pb.update("fetch inventory", 0, 2)
193
to_weave = self.to_control.get_weave('inventory',
194
self.to_repository.get_transaction())
196
child_pb = bzrlib.ui.ui_factory.nested_progress_bar()
198
# just merge, this is optimisable and its means we dont
199
# copy unreferenced data such as not-needed inventories.
200
pb.update("fetch inventory", 1, 3)
201
from_weave = self.from_repository.get_inventory_weave()
202
pb.update("fetch inventory", 2, 3)
203
# we fetch only the referenced inventories because we do not
204
# know for unselected inventories whether all their required
205
# texts are present in the other repository - it could be
207
to_weave.join(from_weave, pb=child_pb, msg='merge inventory',
202
def _fetch_weave_text(self, file_id, required_versions):
203
to_weave = self.to_weaves.get_weave_or_empty(file_id,
204
self.to_repository.get_transaction())
205
from_weave = self.from_weaves.get_weave(file_id,
206
self.from_repository.get_transaction())
207
# we fetch all the texts, because texts do
208
# not reference anything, and its cheap enough
209
to_weave.join(from_weave, version_ids=required_versions)
210
# we don't need *all* of this data anymore, but we dont know
211
# what we do. This cache clearing will result in a new read
212
# of the knit data when we do the checkout, but probably we
213
# want to emit the needed data on the fly rather than at the
215
# the from weave should know not to cache data being joined,
216
# but its ok to ask it to clear.
217
from_weave.clear_cache()
218
to_weave.clear_cache()
220
def _fetch_inventory_weave(self, revs, pb):
221
pb.update("fetch inventory", 0, 2)
222
to_weave = self.to_repository.get_inventory_weave()
223
child_pb = bzrlib.ui.ui_factory.nested_progress_bar()
225
# just merge, this is optimisable and its means we don't
226
# copy unreferenced data such as not-needed inventories.
227
pb.update("fetch inventory", 1, 3)
228
from_weave = self.from_repository.get_inventory_weave()
229
pb.update("fetch inventory", 2, 3)
230
# we fetch only the referenced inventories because we do not
231
# know for unselected inventories whether all their required
232
# texts are present in the other repository - it could be
234
to_weave.join(from_weave, pb=child_pb, msg='merge inventory',
236
from_weave.clear_cache()
240
def _generate_root_texts(self, revs):
241
"""This will be called by __fetch between fetching weave texts and
242
fetching the inventory weave.
244
Subclasses should override this if they need to generate root texts
245
after fetching weave texts.
215
250
class GenericRepoFetcher(RepoFetcher):
219
254
It triggers a reconciliation after fetching to ensure integrity.
222
def _fetch_revision_texts(self, revs):
257
def _fetch_revision_texts(self, revs, pb):
223
258
"""Fetch revision object texts"""
224
rev_pb = bzrlib.ui.ui_factory.nested_progress_bar()
226
to_txn = self.to_transaction = self.to_repository.get_transaction()
229
to_store = self.to_repository._revision_store
231
pb = bzrlib.ui.ui_factory.nested_progress_bar()
233
pb.update('copying revisions', count, total)
235
sig_text = self.from_repository.get_signature_text(rev)
236
to_store.add_revision_signature_text(rev, sig_text, to_txn)
237
except errors.NoSuchRevision:
240
to_store.add_revision(self.from_repository.get_revision(rev),
245
# fixup inventory if needed:
246
# this is expensive because we have no inverse index to current ghosts.
247
# but on local disk its a few seconds and sftp push is already insane.
249
# FIXME: repository should inform if this is needed.
250
self.to_repository.reconcile()
259
to_txn = self.to_transaction = self.to_repository.get_transaction()
262
to_store = self.to_repository._revision_store
264
pb.update('copying revisions', count, total)
266
sig_text = self.from_repository.get_signature_text(rev)
267
to_store.add_revision_signature_text(rev, sig_text, to_txn)
268
except errors.NoSuchRevision:
271
to_store.add_revision(self.from_repository.get_revision(rev),
274
# fixup inventory if needed:
275
# this is expensive because we have no inverse index to current ghosts.
276
# but on local disk its a few seconds and sftp push is already insane.
278
# FIXME: repository should inform if this is needed.
279
self.to_repository.reconcile()
255
282
class KnitRepoFetcher(RepoFetcher):
276
303
to_rf.join(from_rf, version_ids=revs)
279
class Fetcher(object):
280
"""Backwards compatability glue for branch.fetch()."""
282
@deprecated_method(zero_eight)
283
def __init__(self, to_branch, from_branch, last_revision=None, pb=None):
284
"""Please see branch.fetch()."""
285
to_branch.fetch(from_branch, last_revision, pb)
306
class Inter1and2Helper(object):
307
"""Helper for operations that convert data from model 1 and 2
309
This is for use by fetchers and converters.
312
def __init__(self, source, target):
315
:param source: The repository data comes from
316
:param target: The repository data goes to
321
def iter_rev_trees(self, revs):
322
"""Iterate through RevisionTrees efficiently.
324
Additionally, the inventory's revision_id is set if unset.
326
Trees are retrieved in batches of 100, and then yielded in the order
329
:param revs: A list of revision ids
332
for tree in self.source.revision_trees(revs[:100]):
333
if tree.inventory.revision_id is None:
334
tree.inventory.revision_id = tree.get_revision_id()
338
def generate_root_texts(self, revs):
339
"""Generate VersionedFiles for all root ids.
341
:param revs: the revisions to include
343
inventory_weave = self.source.get_inventory_weave()
346
to_store = self.target.weave_store
347
for tree in self.iter_rev_trees(revs):
348
revision_id = tree.inventory.root.revision
349
root_id = tree.inventory.root.file_id
350
parents = inventory_weave.get_parents(revision_id)
351
if root_id not in versionedfile:
352
versionedfile[root_id] = to_store.get_weave_or_empty(root_id,
353
self.target.get_transaction())
354
_, _, parent_texts[root_id] = versionedfile[root_id].add_lines(
355
revision_id, parents, [], parent_texts)
357
def regenerate_inventory(self, revs):
358
"""Generate a new inventory versionedfile in target, convertin data.
360
The inventory is retrieved from the source, (deserializing it), and
361
stored in the target (reserializing it in a different format).
362
:param revs: The revisions to include
364
inventory_weave = self.source.get_inventory_weave()
365
for tree in self.iter_rev_trees(revs):
366
parents = inventory_weave.get_parents(tree.get_revision_id())
367
self.target.add_inventory(tree.get_revision_id(), tree.inventory,
371
class Model1toKnit2Fetcher(GenericRepoFetcher):
372
"""Fetch from a Model1 repository into a Knit2 repository
374
def __init__(self, to_repository, from_repository, last_revision=None,
376
self.helper = Inter1and2Helper(from_repository, to_repository)
377
GenericRepoFetcher.__init__(self, to_repository, from_repository,
380
def _generate_root_texts(self, revs):
381
self.helper.generate_root_texts(revs)
383
def _fetch_inventory_weave(self, revs, pb):
384
self.helper.regenerate_inventory(revs)
387
class Knit1to2Fetcher(KnitRepoFetcher):
388
"""Fetch from a Knit1 repository into a Knit2 repository"""
390
def __init__(self, to_repository, from_repository, last_revision=None,
392
self.helper = Inter1and2Helper(from_repository, to_repository)
393
KnitRepoFetcher.__init__(self, to_repository, from_repository,
396
def _generate_root_texts(self, revs):
397
self.helper.generate_root_texts(revs)
399
def _fetch_inventory_weave(self, revs, pb):
400
self.helper.regenerate_inventory(revs)
added
removed
bzrlib/fetch.py
