1
# Copyright (C) 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
# TODO: At some point, handle upgrades by just passing the whole request
18
# across to run on the server.
21
from cStringIO import StringIO
33
from bzrlib.branch import BranchReferenceFormat
34
from bzrlib.bzrdir import BzrDir, RemoteBzrDirFormat
35
from bzrlib.config import BranchConfig, TreeConfig
36
from bzrlib.decorators import needs_read_lock, needs_write_lock
37
from bzrlib.errors import (
41
from bzrlib.lockable_files import LockableFiles
42
from bzrlib.pack import ContainerPushParser
43
from bzrlib.smart import client, vfs
44
from bzrlib.revision import ensure_null, NULL_REVISION
45
from bzrlib.trace import mutter, note, warning
47
# Note: RemoteBzrDirFormat is in bzrdir.py
49
class RemoteBzrDir(BzrDir):
50
"""Control directory on a remote server, accessed via bzr:// or similar."""
52
def __init__(self, transport, _client=None):
53
"""Construct a RemoteBzrDir.
55
:param _client: Private parameter for testing. Disables probing and the
58
BzrDir.__init__(self, transport, RemoteBzrDirFormat())
59
# this object holds a delegated bzrdir that uses file-level operations
60
# to talk to the other side
61
self._real_bzrdir = None
64
medium = transport.get_smart_medium()
65
self._client = client._SmartClient(medium, transport.base)
67
self._client = _client
70
path = self._path_for_remote_call(self._client)
71
response = self._client.call('BzrDir.open', path)
72
if response not in [('yes',), ('no',)]:
73
raise errors.UnexpectedSmartServerResponse(response)
74
if response == ('no',):
75
raise errors.NotBranchError(path=transport.base)
77
def _ensure_real(self):
78
"""Ensure that there is a _real_bzrdir set.
80
Used before calls to self._real_bzrdir.
82
if not self._real_bzrdir:
83
self._real_bzrdir = BzrDir.open_from_transport(
84
self.root_transport, _server_formats=False)
86
def create_repository(self, shared=False):
88
self._real_bzrdir.create_repository(shared=shared)
89
return self.open_repository()
91
def destroy_repository(self):
92
"""See BzrDir.destroy_repository"""
94
self._real_bzrdir.destroy_repository()
96
def create_branch(self):
98
real_branch = self._real_bzrdir.create_branch()
99
return RemoteBranch(self, self.find_repository(), real_branch)
101
def destroy_branch(self):
102
"""See BzrDir.destroy_branch"""
104
self._real_bzrdir.destroy_branch()
106
def create_workingtree(self, revision_id=None, from_branch=None):
107
raise errors.NotLocalUrl(self.transport.base)
109
def find_branch_format(self):
110
"""Find the branch 'format' for this bzrdir.
112
This might be a synthetic object for e.g. RemoteBranch and SVN.
114
b = self.open_branch()
117
def get_branch_reference(self):
118
"""See BzrDir.get_branch_reference()."""
119
path = self._path_for_remote_call(self._client)
121
response = self._client.call('BzrDir.open_branch', path)
122
except errors.ErrorFromSmartServer, err:
123
if err.error_tuple == ('nobranch',):
124
raise errors.NotBranchError(path=self.root_transport.base)
126
if response[0] == 'ok':
127
if response[1] == '':
128
# branch at this location.
131
# a branch reference, use the existing BranchReference logic.
134
raise errors.UnexpectedSmartServerResponse(response)
136
def _get_tree_branch(self):
137
"""See BzrDir._get_tree_branch()."""
138
return None, self.open_branch()
140
def open_branch(self, _unsupported=False):
142
raise NotImplementedError('unsupported flag support not implemented yet.')
143
reference_url = self.get_branch_reference()
144
if reference_url is None:
145
# branch at this location.
146
return RemoteBranch(self, self.find_repository())
148
# a branch reference, use the existing BranchReference logic.
149
format = BranchReferenceFormat()
150
return format.open(self, _found=True, location=reference_url)
152
def open_repository(self):
153
path = self._path_for_remote_call(self._client)
154
verb = 'BzrDir.find_repositoryV2'
157
response = self._client.call(verb, path)
158
except errors.UnknownSmartMethod:
159
verb = 'BzrDir.find_repository'
160
response = self._client.call(verb, path)
161
except errors.ErrorFromSmartServer, err:
162
if err.error_tuple[0] == 'norepository':
163
raise errors.NoRepositoryPresent(self)
165
if response[0] != 'ok':
166
raise errors.UnexpectedSmartServerResponse(response)
167
if verb == 'BzrDir.find_repository':
168
# servers that don't support the V2 method don't support external
170
response = response + ('no', )
171
if not (len(response) == 5):
172
raise SmartProtocolError('incorrect response length %s' % (response,))
173
if response[1] == '':
174
format = RemoteRepositoryFormat()
175
format.rich_root_data = (response[2] == 'yes')
176
format.supports_tree_reference = (response[3] == 'yes')
177
# No wire format to check this yet.
178
format.supports_external_lookups = (response[4] == 'yes')
179
return RemoteRepository(self, format)
181
raise errors.NoRepositoryPresent(self)
183
def open_workingtree(self, recommend_upgrade=True):
185
if self._real_bzrdir.has_workingtree():
186
raise errors.NotLocalUrl(self.root_transport)
188
raise errors.NoWorkingTree(self.root_transport.base)
190
def _path_for_remote_call(self, client):
191
"""Return the path to be used for this bzrdir in a remote call."""
192
return client.remote_path_from_transport(self.root_transport)
194
def get_branch_transport(self, branch_format):
196
return self._real_bzrdir.get_branch_transport(branch_format)
198
def get_repository_transport(self, repository_format):
200
return self._real_bzrdir.get_repository_transport(repository_format)
202
def get_workingtree_transport(self, workingtree_format):
204
return self._real_bzrdir.get_workingtree_transport(workingtree_format)
206
def can_convert_format(self):
207
"""Upgrading of remote bzrdirs is not supported yet."""
210
def needs_format_conversion(self, format=None):
211
"""Upgrading of remote bzrdirs is not supported yet."""
214
def clone(self, url, revision_id=None, force_new_repo=False):
216
return self._real_bzrdir.clone(url, revision_id=revision_id,
217
force_new_repo=force_new_repo)
220
class RemoteRepositoryFormat(repository.RepositoryFormat):
221
"""Format for repositories accessed over a _SmartClient.
223
Instances of this repository are represented by RemoteRepository
226
The RemoteRepositoryFormat is parameterized during construction
227
to reflect the capabilities of the real, remote format. Specifically
228
the attributes rich_root_data and supports_tree_reference are set
229
on a per instance basis, and are not set (and should not be) at
233
_matchingbzrdir = RemoteBzrDirFormat
235
def initialize(self, a_bzrdir, shared=False):
236
if not isinstance(a_bzrdir, RemoteBzrDir):
237
raise AssertionError('%r is not a RemoteBzrDir' % (a_bzrdir,))
238
return a_bzrdir.create_repository(shared=shared)
240
def open(self, a_bzrdir):
241
if not isinstance(a_bzrdir, RemoteBzrDir):
242
raise AssertionError('%r is not a RemoteBzrDir' % (a_bzrdir,))
243
return a_bzrdir.open_repository()
245
def get_format_description(self):
246
return 'bzr remote repository'
248
def __eq__(self, other):
249
return self.__class__ == other.__class__
251
def check_conversion_target(self, target_format):
252
if self.rich_root_data and not target_format.rich_root_data:
253
raise errors.BadConversionTarget(
254
'Does not support rich root data.', target_format)
255
if (self.supports_tree_reference and
256
not getattr(target_format, 'supports_tree_reference', False)):
257
raise errors.BadConversionTarget(
258
'Does not support nested trees', target_format)
261
class RemoteRepository(object):
262
"""Repository accessed over rpc.
264
For the moment most operations are performed using local transport-backed
268
def __init__(self, remote_bzrdir, format, real_repository=None, _client=None):
269
"""Create a RemoteRepository instance.
271
:param remote_bzrdir: The bzrdir hosting this repository.
272
:param format: The RemoteFormat object to use.
273
:param real_repository: If not None, a local implementation of the
274
repository logic for the repository, usually accessing the data
276
:param _client: Private testing parameter - override the smart client
277
to be used by the repository.
280
self._real_repository = real_repository
282
self._real_repository = None
283
self.bzrdir = remote_bzrdir
285
self._client = remote_bzrdir._client
287
self._client = _client
288
self._format = format
289
self._lock_mode = None
290
self._lock_token = None
292
self._leave_lock = False
293
# A cache of looked up revision parent data; reset at unlock time.
294
self._parents_map = None
295
if 'hpss' in debug.debug_flags:
296
self._requested_parents = None
298
# These depend on the actual remote format, so force them off for
299
# maximum compatibility. XXX: In future these should depend on the
300
# remote repository instance, but this is irrelevant until we perform
301
# reconcile via an RPC call.
302
self._reconcile_does_inventory_gc = False
303
self._reconcile_fixes_text_parents = False
304
self._reconcile_backsup_inventory = False
305
self.base = self.bzrdir.transport.base
308
return "%s(%s)" % (self.__class__.__name__, self.base)
312
def abort_write_group(self):
313
"""Complete a write group on the decorated repository.
315
Smart methods peform operations in a single step so this api
316
is not really applicable except as a compatibility thunk
317
for older plugins that don't use e.g. the CommitBuilder
321
return self._real_repository.abort_write_group()
323
def commit_write_group(self):
324
"""Complete a write group on the decorated repository.
326
Smart methods peform operations in a single step so this api
327
is not really applicable except as a compatibility thunk
328
for older plugins that don't use e.g. the CommitBuilder
332
return self._real_repository.commit_write_group()
334
def _ensure_real(self):
335
"""Ensure that there is a _real_repository set.
337
Used before calls to self._real_repository.
339
if not self._real_repository:
340
self.bzrdir._ensure_real()
341
#self._real_repository = self.bzrdir._real_bzrdir.open_repository()
342
self._set_real_repository(self.bzrdir._real_bzrdir.open_repository())
344
def find_text_key_references(self):
345
"""Find the text key references within the repository.
347
:return: a dictionary mapping (file_id, revision_id) tuples to altered file-ids to an iterable of
348
revision_ids. Each altered file-ids has the exact revision_ids that
349
altered it listed explicitly.
350
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
351
to whether they were referred to by the inventory of the
352
revision_id that they contain. The inventory texts from all present
353
revision ids are assessed to generate this report.
356
return self._real_repository.find_text_key_references()
358
def _generate_text_key_index(self):
359
"""Generate a new text key index for the repository.
361
This is an expensive function that will take considerable time to run.
363
:return: A dict mapping (file_id, revision_id) tuples to a list of
364
parents, also (file_id, revision_id) tuples.
367
return self._real_repository._generate_text_key_index()
369
@symbol_versioning.deprecated_method(symbol_versioning.one_four)
370
def get_revision_graph(self, revision_id=None):
371
"""See Repository.get_revision_graph()."""
372
return self._get_revision_graph(revision_id)
374
def _get_revision_graph(self, revision_id):
375
"""Private method for using with old (< 1.2) servers to fallback."""
376
if revision_id is None:
378
elif revision.is_null(revision_id):
381
path = self.bzrdir._path_for_remote_call(self._client)
383
response = self._client.call_expecting_body(
384
'Repository.get_revision_graph', path, revision_id)
385
except errors.ErrorFromSmartServer, err:
386
if err.error_tuple[0] == 'nosuchrevision':
387
raise NoSuchRevision(self, revision_id)
388
if response[0][0] != 'ok':
389
raise errors.UnexpectedSmartServerResponse(response[0])
390
coded = response[1].read_body_bytes()
392
# no revisions in this repository!
394
lines = coded.split('\n')
397
d = tuple(line.split())
398
revision_graph[d[0]] = d[1:]
400
return revision_graph
402
def has_revision(self, revision_id):
403
"""See Repository.has_revision()."""
404
if revision_id == NULL_REVISION:
405
# The null revision is always present.
407
path = self.bzrdir._path_for_remote_call(self._client)
408
response = self._client.call('Repository.has_revision', path, revision_id)
409
if response[0] not in ('yes', 'no'):
410
raise SmartProtocolError('unexpected response code %s' % (response,))
411
return response[0] == 'yes'
413
def has_revisions(self, revision_ids):
414
"""See Repository.has_revisions()."""
416
for revision_id in revision_ids:
417
if self.has_revision(revision_id):
418
result.add(revision_id)
421
def has_same_location(self, other):
422
return (self.__class__ == other.__class__ and
423
self.bzrdir.transport.base == other.bzrdir.transport.base)
425
def get_graph(self, other_repository=None):
426
"""Return the graph for this repository format"""
427
parents_provider = self
428
if (other_repository is not None and
429
other_repository.bzrdir.transport.base !=
430
self.bzrdir.transport.base):
431
parents_provider = graph._StackedParentsProvider(
432
[parents_provider, other_repository._make_parents_provider()])
433
return graph.Graph(parents_provider)
435
def gather_stats(self, revid=None, committers=None):
436
"""See Repository.gather_stats()."""
437
path = self.bzrdir._path_for_remote_call(self._client)
438
# revid can be None to indicate no revisions, not just NULL_REVISION
439
if revid is None or revision.is_null(revid):
443
if committers is None or not committers:
444
fmt_committers = 'no'
446
fmt_committers = 'yes'
447
response = self._client.call_expecting_body(
448
'Repository.gather_stats', path, fmt_revid, fmt_committers)
449
if response[0][0] != 'ok':
450
raise SmartProtocolError('unexpected response code %s'
453
body = response[1].read_body_bytes()
455
for line in body.split('\n'):
458
key, val_text = line.split(':')
459
if key in ('revisions', 'size', 'committers'):
460
result[key] = int(val_text)
461
elif key in ('firstrev', 'latestrev'):
462
values = val_text.split(' ')[1:]
463
result[key] = (float(values[0]), long(values[1]))
467
def find_branches(self, using=False):
468
"""See Repository.find_branches()."""
469
# should be an API call to the server.
471
return self._real_repository.find_branches(using=using)
473
def get_physical_lock_status(self):
474
"""See Repository.get_physical_lock_status()."""
475
# should be an API call to the server.
477
return self._real_repository.get_physical_lock_status()
479
def is_in_write_group(self):
480
"""Return True if there is an open write group.
482
write groups are only applicable locally for the smart server..
484
if self._real_repository:
485
return self._real_repository.is_in_write_group()
488
return self._lock_count >= 1
491
"""See Repository.is_shared()."""
492
path = self.bzrdir._path_for_remote_call(self._client)
493
response = self._client.call('Repository.is_shared', path)
494
if response[0] not in ('yes', 'no'):
495
raise SmartProtocolError('unexpected response code %s' % (response,))
496
return response[0] == 'yes'
498
def is_write_locked(self):
499
return self._lock_mode == 'w'
502
# wrong eventually - want a local lock cache context
503
if not self._lock_mode:
504
self._lock_mode = 'r'
506
self._parents_map = {}
507
if 'hpss' in debug.debug_flags:
508
self._requested_parents = set()
509
if self._real_repository is not None:
510
self._real_repository.lock_read()
512
self._lock_count += 1
514
def _remote_lock_write(self, token):
515
path = self.bzrdir._path_for_remote_call(self._client)
519
response = self._client.call('Repository.lock_write', path, token)
520
except errors.ErrorFromSmartServer, err:
521
if err.error_tuple[0] == 'LockContention':
522
raise errors.LockContention('(remote lock)')
523
elif err.error_tuple[0] == 'UnlockableTransport':
524
raise errors.UnlockableTransport(self.bzrdir.root_transport)
525
elif err.error_tuple[0] == 'LockFailed':
526
raise errors.LockFailed(err.error_tuple[1], err.error_tuple[2])
529
if response[0] == 'ok':
533
raise errors.UnexpectedSmartServerResponse(response)
535
def lock_write(self, token=None):
536
if not self._lock_mode:
537
self._lock_token = self._remote_lock_write(token)
538
# if self._lock_token is None, then this is something like packs or
539
# svn where we don't get to lock the repo, or a weave style repository
540
# where we cannot lock it over the wire and attempts to do so will
542
if self._real_repository is not None:
543
self._real_repository.lock_write(token=self._lock_token)
544
if token is not None:
545
self._leave_lock = True
547
self._leave_lock = False
548
self._lock_mode = 'w'
550
self._parents_map = {}
551
if 'hpss' in debug.debug_flags:
552
self._requested_parents = set()
553
elif self._lock_mode == 'r':
554
raise errors.ReadOnlyError(self)
556
self._lock_count += 1
557
return self._lock_token or None
559
def leave_lock_in_place(self):
560
if not self._lock_token:
561
raise NotImplementedError(self.leave_lock_in_place)
562
self._leave_lock = True
564
def dont_leave_lock_in_place(self):
565
if not self._lock_token:
566
raise NotImplementedError(self.dont_leave_lock_in_place)
567
self._leave_lock = False
569
def _set_real_repository(self, repository):
570
"""Set the _real_repository for this repository.
572
:param repository: The repository to fallback to for non-hpss
573
implemented operations.
575
if isinstance(repository, RemoteRepository):
576
raise AssertionError()
577
self._real_repository = repository
578
if self._lock_mode == 'w':
579
# if we are already locked, the real repository must be able to
580
# acquire the lock with our token.
581
self._real_repository.lock_write(self._lock_token)
582
elif self._lock_mode == 'r':
583
self._real_repository.lock_read()
585
def start_write_group(self):
586
"""Start a write group on the decorated repository.
588
Smart methods peform operations in a single step so this api
589
is not really applicable except as a compatibility thunk
590
for older plugins that don't use e.g. the CommitBuilder
594
return self._real_repository.start_write_group()
596
def _unlock(self, token):
597
path = self.bzrdir._path_for_remote_call(self._client)
599
# with no token the remote repository is not persistently locked.
602
response = self._client.call('Repository.unlock', path, token)
603
except errors.ErrorFromSmartServer, err:
604
if err.error_tuple[0] == 'TokenMismatch':
605
raise errors.TokenMismatch(token, '(remote token)')
607
if response == ('ok',):
610
raise errors.UnexpectedSmartServerResponse(response)
613
self._lock_count -= 1
614
if self._lock_count > 0:
616
self._parents_map = None
617
if 'hpss' in debug.debug_flags:
618
self._requested_parents = None
619
old_mode = self._lock_mode
620
self._lock_mode = None
622
# The real repository is responsible at present for raising an
623
# exception if it's in an unfinished write group. However, it
624
# normally will *not* actually remove the lock from disk - that's
625
# done by the server on receiving the Repository.unlock call.
626
# This is just to let the _real_repository stay up to date.
627
if self._real_repository is not None:
628
self._real_repository.unlock()
630
# The rpc-level lock should be released even if there was a
631
# problem releasing the vfs-based lock.
633
# Only write-locked repositories need to make a remote method
634
# call to perfom the unlock.
635
old_token = self._lock_token
636
self._lock_token = None
637
if not self._leave_lock:
638
self._unlock(old_token)
640
def break_lock(self):
641
# should hand off to the network
643
return self._real_repository.break_lock()
645
def _get_tarball(self, compression):
646
"""Return a TemporaryFile containing a repository tarball.
648
Returns None if the server does not support sending tarballs.
651
path = self.bzrdir._path_for_remote_call(self._client)
653
response, protocol = self._client.call_expecting_body(
654
'Repository.tarball', path, compression)
655
except errors.UnknownSmartMethod:
656
protocol.cancel_read_body()
658
if response[0] == 'ok':
659
# Extract the tarball and return it
660
t = tempfile.NamedTemporaryFile()
661
# TODO: rpc layer should read directly into it...
662
t.write(protocol.read_body_bytes())
665
raise errors.UnexpectedSmartServerResponse(response)
667
def sprout(self, to_bzrdir, revision_id=None):
668
# TODO: Option to control what format is created?
670
dest_repo = self._real_repository._format.initialize(to_bzrdir,
672
dest_repo.fetch(self, revision_id=revision_id)
675
### These methods are just thin shims to the VFS object for now.
677
def revision_tree(self, revision_id):
679
return self._real_repository.revision_tree(revision_id)
681
def get_serializer_format(self):
683
return self._real_repository.get_serializer_format()
685
def get_commit_builder(self, branch, parents, config, timestamp=None,
686
timezone=None, committer=None, revprops=None,
688
# FIXME: It ought to be possible to call this without immediately
689
# triggering _ensure_real. For now it's the easiest thing to do.
691
builder = self._real_repository.get_commit_builder(branch, parents,
692
config, timestamp=timestamp, timezone=timezone,
693
committer=committer, revprops=revprops, revision_id=revision_id)
696
def add_inventory(self, revid, inv, parents):
698
return self._real_repository.add_inventory(revid, inv, parents)
700
def add_revision(self, rev_id, rev, inv=None, config=None):
702
return self._real_repository.add_revision(
703
rev_id, rev, inv=inv, config=config)
706
def get_inventory(self, revision_id):
708
return self._real_repository.get_inventory(revision_id)
710
def iter_inventories(self, revision_ids):
712
return self._real_repository.iter_inventories(revision_ids)
715
def get_revision(self, revision_id):
717
return self._real_repository.get_revision(revision_id)
720
def weave_store(self):
722
return self._real_repository.weave_store
724
def get_transaction(self):
726
return self._real_repository.get_transaction()
729
def clone(self, a_bzrdir, revision_id=None):
731
return self._real_repository.clone(a_bzrdir, revision_id=revision_id)
733
def make_working_trees(self):
734
"""See Repository.make_working_trees"""
736
return self._real_repository.make_working_trees()
738
def revision_ids_to_search_result(self, result_set):
739
"""Convert a set of revision ids to a graph SearchResult."""
740
result_parents = set()
741
for parents in self.get_graph().get_parent_map(
742
result_set).itervalues():
743
result_parents.update(parents)
744
included_keys = result_set.intersection(result_parents)
745
start_keys = result_set.difference(included_keys)
746
exclude_keys = result_parents.difference(result_set)
747
result = graph.SearchResult(start_keys, exclude_keys,
748
len(result_set), result_set)
752
def search_missing_revision_ids(self, other, revision_id=None, find_ghosts=True):
753
"""Return the revision ids that other has that this does not.
755
These are returned in topological order.
757
revision_id: only return revision ids included by revision_id.
759
return repository.InterRepository.get(
760
other, self).search_missing_revision_ids(revision_id, find_ghosts)
762
def fetch(self, source, revision_id=None, pb=None):
763
if self.has_same_location(source):
764
# check that last_revision is in 'from' and then return a
766
if (revision_id is not None and
767
not revision.is_null(revision_id)):
768
self.get_revision(revision_id)
771
return self._real_repository.fetch(
772
source, revision_id=revision_id, pb=pb)
774
def create_bundle(self, target, base, fileobj, format=None):
776
self._real_repository.create_bundle(target, base, fileobj, format)
779
def control_weaves(self):
781
return self._real_repository.control_weaves
784
def get_ancestry(self, revision_id, topo_sorted=True):
786
return self._real_repository.get_ancestry(revision_id, topo_sorted)
789
def get_inventory_weave(self):
791
return self._real_repository.get_inventory_weave()
793
def fileids_altered_by_revision_ids(self, revision_ids):
795
return self._real_repository.fileids_altered_by_revision_ids(revision_ids)
797
def _get_versioned_file_checker(self, revisions, revision_versions_cache):
799
return self._real_repository._get_versioned_file_checker(
800
revisions, revision_versions_cache)
802
def iter_files_bytes(self, desired_files):
803
"""See Repository.iter_file_bytes.
806
return self._real_repository.iter_files_bytes(desired_files)
808
def get_parent_map(self, keys):
809
"""See bzrlib.Graph.get_parent_map()."""
810
# Hack to build up the caching logic.
811
ancestry = self._parents_map
813
# Repository is not locked, so there's no cache.
814
missing_revisions = set(keys)
817
missing_revisions = set(key for key in keys if key not in ancestry)
818
if missing_revisions:
819
parent_map = self._get_parent_map(missing_revisions)
820
if 'hpss' in debug.debug_flags:
821
mutter('retransmitted revisions: %d of %d',
822
len(set(ancestry).intersection(parent_map)),
824
ancestry.update(parent_map)
825
present_keys = [k for k in keys if k in ancestry]
826
if 'hpss' in debug.debug_flags:
827
if self._requested_parents is not None and len(ancestry) != 0:
828
self._requested_parents.update(present_keys)
829
mutter('Current RemoteRepository graph hit rate: %d%%',
830
100.0 * len(self._requested_parents) / len(ancestry))
831
return dict((k, ancestry[k]) for k in present_keys)
833
def _get_parent_map(self, keys):
834
"""Helper for get_parent_map that performs the RPC."""
835
medium = self._client._medium
836
if not medium._remote_is_at_least_1_2:
837
# We already found out that the server can't understand
838
# Repository.get_parent_map requests, so just fetch the whole
840
# XXX: Note that this will issue a deprecation warning. This is ok
841
# :- its because we're working with a deprecated server anyway, and
842
# the user will almost certainly have seen a warning about the
843
# server version already.
844
rg = self.get_revision_graph()
845
# There is an api discrepency between get_parent_map and
846
# get_revision_graph. Specifically, a "key:()" pair in
847
# get_revision_graph just means a node has no parents. For
848
# "get_parent_map" it means the node is a ghost. So fix up the
849
# graph to correct this.
850
# https://bugs.launchpad.net/bzr/+bug/214894
851
# There is one other "bug" which is that ghosts in
852
# get_revision_graph() are not returned at all. But we won't worry
853
# about that for now.
854
for node_id, parent_ids in rg.iteritems():
856
rg[node_id] = (NULL_REVISION,)
857
rg[NULL_REVISION] = ()
861
if NULL_REVISION in keys:
862
keys.discard(NULL_REVISION)
863
found_parents = {NULL_REVISION:()}
868
# TODO(Needs analysis): We could assume that the keys being requested
869
# from get_parent_map are in a breadth first search, so typically they
870
# will all be depth N from some common parent, and we don't have to
871
# have the server iterate from the root parent, but rather from the
872
# keys we're searching; and just tell the server the keyspace we
873
# already have; but this may be more traffic again.
875
# Transform self._parents_map into a search request recipe.
876
# TODO: Manage this incrementally to avoid covering the same path
877
# repeatedly. (The server will have to on each request, but the less
878
# work done the better).
879
parents_map = self._parents_map
880
if parents_map is None:
881
# Repository is not locked, so there's no cache.
883
start_set = set(parents_map)
884
result_parents = set()
885
for parents in parents_map.itervalues():
886
result_parents.update(parents)
887
stop_keys = result_parents.difference(start_set)
888
included_keys = start_set.intersection(result_parents)
889
start_set.difference_update(included_keys)
890
recipe = (start_set, stop_keys, len(parents_map))
891
body = self._serialise_search_recipe(recipe)
892
path = self.bzrdir._path_for_remote_call(self._client)
894
if type(key) is not str:
896
"key %r not a plain string" % (key,))
897
verb = 'Repository.get_parent_map'
898
args = (path,) + tuple(keys)
900
response = self._client.call_with_body_bytes_expecting_body(
901
verb, args, self._serialise_search_recipe(recipe))
902
except errors.UnknownSmartMethod:
903
# Server does not support this method, so get the whole graph.
904
# Worse, we have to force a disconnection, because the server now
905
# doesn't realise it has a body on the wire to consume, so the
906
# only way to recover is to abandon the connection.
908
'Server is too old for fast get_parent_map, reconnecting. '
909
'(Upgrade the server to Bazaar 1.2 to avoid this)')
911
# To avoid having to disconnect repeatedly, we keep track of the
912
# fact the server doesn't understand remote methods added in 1.2.
913
medium._remote_is_at_least_1_2 = False
914
return self.get_revision_graph(None)
915
if response[0][0] not in ['ok']:
916
response[1].cancel_read_body()
917
raise errors.UnexpectedSmartServerResponse(response[0])
918
if response[0][0] == 'ok':
919
coded = bz2.decompress(response[1].read_body_bytes())
923
lines = coded.split('\n')
926
d = tuple(line.split())
928
revision_graph[d[0]] = d[1:]
930
# No parents - so give the Graph result (NULL_REVISION,).
931
revision_graph[d[0]] = (NULL_REVISION,)
932
return revision_graph
935
def get_signature_text(self, revision_id):
937
return self._real_repository.get_signature_text(revision_id)
940
@symbol_versioning.deprecated_method(symbol_versioning.one_three)
941
def get_revision_graph_with_ghosts(self, revision_ids=None):
943
return self._real_repository.get_revision_graph_with_ghosts(
944
revision_ids=revision_ids)
947
def get_inventory_xml(self, revision_id):
949
return self._real_repository.get_inventory_xml(revision_id)
951
def deserialise_inventory(self, revision_id, xml):
953
return self._real_repository.deserialise_inventory(revision_id, xml)
955
def reconcile(self, other=None, thorough=False):
957
return self._real_repository.reconcile(other=other, thorough=thorough)
959
def all_revision_ids(self):
961
return self._real_repository.all_revision_ids()
964
def get_deltas_for_revisions(self, revisions):
966
return self._real_repository.get_deltas_for_revisions(revisions)
969
def get_revision_delta(self, revision_id):
971
return self._real_repository.get_revision_delta(revision_id)
974
def revision_trees(self, revision_ids):
976
return self._real_repository.revision_trees(revision_ids)
979
def get_revision_reconcile(self, revision_id):
981
return self._real_repository.get_revision_reconcile(revision_id)
984
def check(self, revision_ids=None):
986
return self._real_repository.check(revision_ids=revision_ids)
988
def copy_content_into(self, destination, revision_id=None):
990
return self._real_repository.copy_content_into(
991
destination, revision_id=revision_id)
993
def _copy_repository_tarball(self, to_bzrdir, revision_id=None):
994
# get a tarball of the remote repository, and copy from that into the
996
from bzrlib import osutils
999
# TODO: Maybe a progress bar while streaming the tarball?
1000
note("Copying repository content as tarball...")
1001
tar_file = self._get_tarball('bz2')
1002
if tar_file is None:
1004
destination = to_bzrdir.create_repository()
1006
tar = tarfile.open('repository', fileobj=tar_file,
1008
tmpdir = tempfile.mkdtemp()
1010
_extract_tar(tar, tmpdir)
1011
tmp_bzrdir = BzrDir.open(tmpdir)
1012
tmp_repo = tmp_bzrdir.open_repository()
1013
tmp_repo.copy_content_into(destination, revision_id)
1015
osutils.rmtree(tmpdir)
1019
# TODO: Suggestion from john: using external tar is much faster than
1020
# python's tarfile library, but it may not work on windows.
1024
"""Compress the data within the repository.
1026
This is not currently implemented within the smart server.
1029
return self._real_repository.pack()
1031
def set_make_working_trees(self, new_value):
1033
self._real_repository.set_make_working_trees(new_value)
1036
def sign_revision(self, revision_id, gpg_strategy):
1038
return self._real_repository.sign_revision(revision_id, gpg_strategy)
1041
def get_revisions(self, revision_ids):
1043
return self._real_repository.get_revisions(revision_ids)
1045
def supports_rich_root(self):
1047
return self._real_repository.supports_rich_root()
1049
def iter_reverse_revision_history(self, revision_id):
1051
return self._real_repository.iter_reverse_revision_history(revision_id)
1054
def _serializer(self):
1056
return self._real_repository._serializer
1058
def store_revision_signature(self, gpg_strategy, plaintext, revision_id):
1060
return self._real_repository.store_revision_signature(
1061
gpg_strategy, plaintext, revision_id)
1063
def add_signature_text(self, revision_id, signature):
1065
return self._real_repository.add_signature_text(revision_id, signature)
1067
def has_signature_for_revision_id(self, revision_id):
1069
return self._real_repository.has_signature_for_revision_id(revision_id)
1071
def get_data_stream_for_search(self, search):
1072
medium = self._client._medium
1073
if not medium._remote_is_at_least_1_2:
1075
return self._real_repository.get_data_stream_for_search(search)
1076
REQUEST_NAME = 'Repository.stream_revisions_chunked'
1077
path = self.bzrdir._path_for_remote_call(self._client)
1078
body = self._serialise_search_recipe(search.get_recipe())
1080
result = self._client.call_with_body_bytes_expecting_body(
1081
REQUEST_NAME, (path,), body)
1082
response, protocol = result
1083
except errors.UnknownSmartMethod:
1084
# Server does not support this method, so fall back to VFS.
1085
# Worse, we have to force a disconnection, because the server now
1086
# doesn't realise it has a body on the wire to consume, so the
1087
# only way to recover is to abandon the connection.
1089
'Server is too old for streaming pull, reconnecting. '
1090
'(Upgrade the server to Bazaar 1.2 to avoid this)')
1092
# To avoid having to disconnect repeatedly, we keep track of the
1093
# fact the server doesn't understand this remote method.
1094
medium._remote_is_at_least_1_2 = False
1096
return self._real_repository.get_data_stream_for_search(search)
1098
if response == ('ok',):
1099
return self._deserialise_stream(protocol)
1100
if response == ('NoSuchRevision', ):
1101
# We cannot easily identify the revision that is missing in this
1102
# situation without doing much more network IO. For now, bail.
1103
raise NoSuchRevision(self, "unknown")
1105
raise errors.UnexpectedSmartServerResponse(response)
1107
def _deserialise_stream(self, protocol):
1108
stream = protocol.read_streamed_body()
1109
container_parser = ContainerPushParser()
1110
for bytes in stream:
1111
container_parser.accept_bytes(bytes)
1112
records = container_parser.read_pending_records()
1113
for record_names, record_bytes in records:
1114
if len(record_names) != 1:
1115
# These records should have only one name, and that name
1116
# should be a one-element tuple.
1117
raise errors.SmartProtocolError(
1118
'Repository data stream had invalid record name %r'
1120
name_tuple = record_names[0]
1121
yield name_tuple, record_bytes
1123
def insert_data_stream(self, stream):
1125
self._real_repository.insert_data_stream(stream)
1127
def item_keys_introduced_by(self, revision_ids, _files_pb=None):
1129
return self._real_repository.item_keys_introduced_by(revision_ids,
1130
_files_pb=_files_pb)
1132
def revision_graph_can_have_wrong_parents(self):
1133
# The answer depends on the remote repo format.
1135
return self._real_repository.revision_graph_can_have_wrong_parents()
1137
def _find_inconsistent_revision_parents(self):
1139
return self._real_repository._find_inconsistent_revision_parents()
1141
def _check_for_inconsistent_revision_parents(self):
1143
return self._real_repository._check_for_inconsistent_revision_parents()
1145
def _make_parents_provider(self):
1148
def _serialise_search_recipe(self, recipe):
1149
"""Serialise a graph search recipe.
1151
:param recipe: A search recipe (start, stop, count).
1152
:return: Serialised bytes.
1154
start_keys = ' '.join(recipe[0])
1155
stop_keys = ' '.join(recipe[1])
1156
count = str(recipe[2])
1157
return '\n'.join((start_keys, stop_keys, count))
1160
class RemoteBranchLockableFiles(LockableFiles):
1161
"""A 'LockableFiles' implementation that talks to a smart server.
1163
This is not a public interface class.
1166
def __init__(self, bzrdir, _client):
1167
self.bzrdir = bzrdir
1168
self._client = _client
1169
self._need_find_modes = True
1170
LockableFiles.__init__(
1171
self, bzrdir.get_branch_transport(None),
1172
'lock', lockdir.LockDir)
1174
def _find_modes(self):
1175
# RemoteBranches don't let the client set the mode of control files.
1176
self._dir_mode = None
1177
self._file_mode = None
1180
class RemoteBranchFormat(branch.BranchFormat):
1182
def __eq__(self, other):
1183
return (isinstance(other, RemoteBranchFormat) and
1184
self.__dict__ == other.__dict__)
1186
def get_format_description(self):
1187
return 'Remote BZR Branch'
1189
def get_format_string(self):
1190
return 'Remote BZR Branch'
1192
def open(self, a_bzrdir):
1193
return a_bzrdir.open_branch()
1195
def initialize(self, a_bzrdir):
1196
return a_bzrdir.create_branch()
1198
def supports_tags(self):
1199
# Remote branches might support tags, but we won't know until we
1200
# access the real remote branch.
1204
class RemoteBranch(branch.Branch):
1205
"""Branch stored on a server accessed by HPSS RPC.
1207
At the moment most operations are mapped down to simple file operations.
1210
def __init__(self, remote_bzrdir, remote_repository, real_branch=None,
1212
"""Create a RemoteBranch instance.
1214
:param real_branch: An optional local implementation of the branch
1215
format, usually accessing the data via the VFS.
1216
:param _client: Private parameter for testing.
1218
# We intentionally don't call the parent class's __init__, because it
1219
# will try to assign to self.tags, which is a property in this subclass.
1220
# And the parent's __init__ doesn't do much anyway.
1221
self._revision_id_to_revno_cache = None
1222
self._revision_history_cache = None
1223
self.bzrdir = remote_bzrdir
1224
if _client is not None:
1225
self._client = _client
1227
self._client = remote_bzrdir._client
1228
self.repository = remote_repository
1229
if real_branch is not None:
1230
self._real_branch = real_branch
1231
# Give the remote repository the matching real repo.
1232
real_repo = self._real_branch.repository
1233
if isinstance(real_repo, RemoteRepository):
1234
real_repo._ensure_real()
1235
real_repo = real_repo._real_repository
1236
self.repository._set_real_repository(real_repo)
1237
# Give the branch the remote repository to let fast-pathing happen.
1238
self._real_branch.repository = self.repository
1240
self._real_branch = None
1241
# Fill out expected attributes of branch for bzrlib api users.
1242
self._format = RemoteBranchFormat()
1243
self.base = self.bzrdir.root_transport.base
1244
self._control_files = None
1245
self._lock_mode = None
1246
self._lock_token = None
1247
self._repo_lock_token = None
1248
self._lock_count = 0
1249
self._leave_lock = False
1252
return "%s(%s)" % (self.__class__.__name__, self.base)
1256
def _ensure_real(self):
1257
"""Ensure that there is a _real_branch set.
1259
Used before calls to self._real_branch.
1261
if not self._real_branch:
1262
if not vfs.vfs_enabled():
1263
raise AssertionError('smart server vfs must be enabled '
1264
'to use vfs implementation')
1265
self.bzrdir._ensure_real()
1266
self._real_branch = self.bzrdir._real_bzrdir.open_branch()
1267
# Give the remote repository the matching real repo.
1268
real_repo = self._real_branch.repository
1269
if isinstance(real_repo, RemoteRepository):
1270
real_repo._ensure_real()
1271
real_repo = real_repo._real_repository
1272
self.repository._set_real_repository(real_repo)
1273
# Give the branch the remote repository to let fast-pathing happen.
1274
self._real_branch.repository = self.repository
1275
# XXX: deal with _lock_mode == 'w'
1276
if self._lock_mode == 'r':
1277
self._real_branch.lock_read()
1280
def control_files(self):
1281
# Defer actually creating RemoteBranchLockableFiles until its needed,
1282
# because it triggers an _ensure_real that we otherwise might not need.
1283
if self._control_files is None:
1284
self._control_files = RemoteBranchLockableFiles(
1285
self.bzrdir, self._client)
1286
return self._control_files
1288
def _get_checkout_format(self):
1290
return self._real_branch._get_checkout_format()
1292
def get_physical_lock_status(self):
1293
"""See Branch.get_physical_lock_status()."""
1294
# should be an API call to the server, as branches must be lockable.
1296
return self._real_branch.get_physical_lock_status()
1298
def lock_read(self):
1299
if not self._lock_mode:
1300
self._lock_mode = 'r'
1301
self._lock_count = 1
1302
if self._real_branch is not None:
1303
self._real_branch.lock_read()
1305
self._lock_count += 1
1307
def _remote_lock_write(self, token):
1309
branch_token = repo_token = ''
1311
branch_token = token
1312
repo_token = self.repository.lock_write()
1313
self.repository.unlock()
1314
path = self.bzrdir._path_for_remote_call(self._client)
1316
response = self._client.call(
1317
'Branch.lock_write', path, branch_token, repo_token or '')
1318
except errors.ErrorFromSmartServer, err:
1319
if err.error_tuple[0] == 'LockContention':
1320
raise errors.LockContention('(remote lock)')
1321
elif err.error_tuple[0] == 'TokenMismatch':
1322
raise errors.TokenMismatch(token, '(remote token)')
1323
elif err.error_tuple[0] == 'UnlockableTransport':
1324
raise errors.UnlockableTransport(self.bzrdir.root_transport)
1325
elif err.error_tuple[0] == 'ReadOnlyError':
1326
raise errors.ReadOnlyError(self)
1327
elif err.error_tuple[0] == 'LockFailed':
1328
raise errors.LockFailed(err.error_tuple[1], err.error_tuple[2])
1330
if response[0] != 'ok':
1331
raise errors.UnexpectedSmartServerResponse(response)
1332
ok, branch_token, repo_token = response
1333
return branch_token, repo_token
1335
def lock_write(self, token=None):
1336
if not self._lock_mode:
1337
remote_tokens = self._remote_lock_write(token)
1338
self._lock_token, self._repo_lock_token = remote_tokens
1339
if not self._lock_token:
1340
raise SmartProtocolError('Remote server did not return a token!')
1341
# TODO: We really, really, really don't want to call _ensure_real
1342
# here, but it's the easiest way to ensure coherency between the
1343
# state of the RemoteBranch and RemoteRepository objects and the
1344
# physical locks. If we don't materialise the real objects here,
1345
# then getting everything in the right state later is complex, so
1346
# for now we just do it the lazy way.
1347
# -- Andrew Bennetts, 2007-02-22.
1349
if self._real_branch is not None:
1350
self._real_branch.repository.lock_write(
1351
token=self._repo_lock_token)
1353
self._real_branch.lock_write(token=self._lock_token)
1355
self._real_branch.repository.unlock()
1356
if token is not None:
1357
self._leave_lock = True
1359
# XXX: this case seems to be unreachable; token cannot be None.
1360
self._leave_lock = False
1361
self._lock_mode = 'w'
1362
self._lock_count = 1
1363
elif self._lock_mode == 'r':
1364
raise errors.ReadOnlyTransaction
1366
if token is not None:
1367
# A token was given to lock_write, and we're relocking, so check
1368
# that the given token actually matches the one we already have.
1369
if token != self._lock_token:
1370
raise errors.TokenMismatch(token, self._lock_token)
1371
self._lock_count += 1
1372
return self._lock_token or None
1374
def _unlock(self, branch_token, repo_token):
1375
path = self.bzrdir._path_for_remote_call(self._client)
1377
response = self._client.call('Branch.unlock', path, branch_token,
1379
except errors.ErrorFromSmartServer, err:
1380
if err.error_tuple[0] == 'TokenMismatch':
1381
raise errors.TokenMismatch(
1382
str((branch_token, repo_token)), '(remote tokens)')
1384
if response == ('ok',):
1386
raise errors.UnexpectedSmartServerResponse(response)
1389
self._lock_count -= 1
1390
if not self._lock_count:
1391
self._clear_cached_state()
1392
mode = self._lock_mode
1393
self._lock_mode = None
1394
if self._real_branch is not None:
1395
if (not self._leave_lock and mode == 'w' and
1396
self._repo_lock_token):
1397
# If this RemoteBranch will remove the physical lock for the
1398
# repository, make sure the _real_branch doesn't do it
1399
# first. (Because the _real_branch's repository is set to
1400
# be the RemoteRepository.)
1401
self._real_branch.repository.leave_lock_in_place()
1402
self._real_branch.unlock()
1404
# Only write-locked branched need to make a remote method call
1405
# to perfom the unlock.
1407
if not self._lock_token:
1408
raise AssertionError('Locked, but no token!')
1409
branch_token = self._lock_token
1410
repo_token = self._repo_lock_token
1411
self._lock_token = None
1412
self._repo_lock_token = None
1413
if not self._leave_lock:
1414
self._unlock(branch_token, repo_token)
1416
def break_lock(self):
1418
return self._real_branch.break_lock()
1420
def leave_lock_in_place(self):
1421
if not self._lock_token:
1422
raise NotImplementedError(self.leave_lock_in_place)
1423
self._leave_lock = True
1425
def dont_leave_lock_in_place(self):
1426
if not self._lock_token:
1427
raise NotImplementedError(self.dont_leave_lock_in_place)
1428
self._leave_lock = False
1430
def last_revision_info(self):
1431
"""See Branch.last_revision_info()."""
1432
path = self.bzrdir._path_for_remote_call(self._client)
1433
response = self._client.call('Branch.last_revision_info', path)
1434
if response[0] != 'ok':
1435
raise SmartProtocolError('unexpected response code %s' % (response,))
1436
revno = int(response[1])
1437
last_revision = response[2]
1438
return (revno, last_revision)
1440
def _gen_revision_history(self):
1441
"""See Branch._gen_revision_history()."""
1442
path = self.bzrdir._path_for_remote_call(self._client)
1443
response = self._client.call_expecting_body(
1444
'Branch.revision_history', path)
1445
if response[0][0] != 'ok':
1446
raise SmartProtocolError('unexpected response code %s' % (response,))
1447
result = response[1].read_body_bytes().split('\x00')
1453
def set_revision_history(self, rev_history):
1454
# Send just the tip revision of the history; the server will generate
1455
# the full history from that. If the revision doesn't exist in this
1456
# branch, NoSuchRevision will be raised.
1457
path = self.bzrdir._path_for_remote_call(self._client)
1458
if rev_history == []:
1461
rev_id = rev_history[-1]
1462
self._clear_cached_state()
1464
response = self._client.call('Branch.set_last_revision',
1465
path, self._lock_token, self._repo_lock_token, rev_id)
1466
except errors.ErrorFromSmartServer, err:
1467
if err.error_tuple[0] == 'NoSuchRevision':
1468
raise NoSuchRevision(self, rev_id)
1470
if response != ('ok',):
1471
raise errors.UnexpectedSmartServerResponse(response)
1472
self._cache_revision_history(rev_history)
1474
def get_parent(self):
1476
return self._real_branch.get_parent()
1478
def set_parent(self, url):
1480
return self._real_branch.set_parent(url)
1482
def sprout(self, to_bzrdir, revision_id=None):
1483
# Like Branch.sprout, except that it sprouts a branch in the default
1484
# format, because RemoteBranches can't be created at arbitrary URLs.
1485
# XXX: if to_bzrdir is a RemoteBranch, this should perhaps do
1486
# to_bzrdir.create_branch...
1488
result = self._real_branch._format.initialize(to_bzrdir)
1489
self.copy_content_into(result, revision_id=revision_id)
1490
result.set_parent(self.bzrdir.root_transport.base)
1494
def pull(self, source, overwrite=False, stop_revision=None,
1496
# FIXME: This asks the real branch to run the hooks, which means
1497
# they're called with the wrong target branch parameter.
1498
# The test suite specifically allows this at present but it should be
1499
# fixed. It should get a _override_hook_target branch,
1500
# as push does. -- mbp 20070405
1502
self._real_branch.pull(
1503
source, overwrite=overwrite, stop_revision=stop_revision,
1507
def push(self, target, overwrite=False, stop_revision=None):
1509
return self._real_branch.push(
1510
target, overwrite=overwrite, stop_revision=stop_revision,
1511
_override_hook_source_branch=self)
1513
def is_locked(self):
1514
return self._lock_count >= 1
1517
def set_last_revision_info(self, revno, revision_id):
1518
revision_id = ensure_null(revision_id)
1519
path = self.bzrdir._path_for_remote_call(self._client)
1521
response = self._client.call('Branch.set_last_revision_info',
1522
path, self._lock_token, self._repo_lock_token, str(revno), revision_id)
1523
except errors.UnknownSmartMethod:
1525
self._clear_cached_state()
1526
return self._real_branch.set_last_revision_info(revno, revision_id)
1527
except errors.ErrorFromSmartServer, err:
1528
if err.error_tuple[0] == 'NoSuchRevision':
1529
raise NoSuchRevision(self, err.error_tuple[1])
1530
if response == ('ok',):
1531
self._clear_cached_state()
1533
raise errors.UnexpectedSmartServerResponse(response)
1535
def generate_revision_history(self, revision_id, last_rev=None,
1538
return self._real_branch.generate_revision_history(
1539
revision_id, last_rev=last_rev, other_branch=other_branch)
1544
return self._real_branch.tags
1546
def set_push_location(self, location):
1548
return self._real_branch.set_push_location(location)
1550
def update_revisions(self, other, stop_revision=None, overwrite=False):
1552
return self._real_branch.update_revisions(
1553
other, stop_revision=stop_revision, overwrite=overwrite)
1556
def _extract_tar(tar, to_dir):
1557
"""Extract all the contents of a tarfile object.
1559
A replacement for extractall, which is not present in python2.4
1562
tar.extract(tarinfo, to_dir)