1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
52
from bzrlib.transport.local import LocalURLServer
58
class FtpPathError(errors.PathError):
59
"""FTP failed for path: %(path)s%(extra)s"""
62
class FtpStatResult(object):
63
def __init__(self, f, relpath):
65
self.st_size = f.size(relpath)
66
self.st_mode = stat.S_IFREG
67
except ftplib.error_perm:
71
self.st_mode = stat.S_IFDIR
76
_number_of_retries = 2
77
_sleep_between_retries = 5
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never
83
# retried. --vila 20070720 (Bug #127164)
84
class FtpTransport(ConnectedTransport):
85
"""This is the transport agent for ftp:// access."""
87
def __init__(self, base, _from_transport=None):
88
"""Set the base path where files will be stored."""
89
assert base.startswith('ftp://') or base.startswith('aftp://')
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
99
"""Return the ftplib.FTP instance for this object."""
100
# Ensures that a connection is established
101
connection = self._get_connection()
102
if connection is None:
103
# First connection ever
104
connection, credentials = self._create_connection()
105
self._set_connection(connection, credentials)
108
def _create_connection(self, credentials=None):
109
"""Create a new connection with the provided credentials.
111
:param credentials: The credentials needed to establish the connection.
113
:return: The created connection and its associated credentials.
115
The credentials are only the password as it may have been entered
116
interactively by the user and may be different from the one provided
117
in base url at transport creation time.
119
if credentials is None:
120
password = self._password
122
password = credentials
124
mutter("Constructing FTP instance against %r" %
125
((self._host, self._port, self._user, '********',
128
connection = ftplib.FTP()
129
connection.connect(host=self._host, port=self._port)
130
if self._user and self._user != 'anonymous' and \
131
password is not None: # '' is a valid password
132
get_password = bzrlib.ui.ui_factory.get_password
133
password = get_password(prompt='FTP %(user)s@%(host)s password',
134
user=self._user, host=self._host)
135
connection.login(user=self._user, passwd=password)
136
connection.set_pasv(not self.is_active)
137
except ftplib.error_perm, e:
138
raise errors.TransportError(msg="Error setting up connection:"
139
" %s" % str(e), orig_error=e)
140
return connection, password
142
def _reconnect(self):
143
"""Create a new connection with the previously used credentials"""
144
credentials = self.get_credentials()
145
connection, credentials = self._create_connection(credentials)
146
self._set_connection(connection, credentials)
148
def _translate_perm_error(self, err, path, extra=None,
149
unknown_exc=FtpPathError):
150
"""Try to translate an ftplib.error_perm exception.
152
:param err: The error to translate into a bzr error
153
:param path: The path which had problems
154
:param extra: Extra information which can be included
155
:param unknown_exc: If None, we will just raise the original exception
156
otherwise we raise unknown_exc(path, extra=extra)
162
extra += ': ' + str(err)
163
if ('no such file' in s
164
or 'could not open' in s
165
or 'no such dir' in s
166
or 'could not create file' in s # vsftpd
167
or 'file doesn\'t exist' in s
169
raise errors.NoSuchFile(path, extra=extra)
170
if ('file exists' in s):
171
raise errors.FileExists(path, extra=extra)
172
if ('not a directory' in s):
173
raise errors.PathError(path, extra=extra)
175
mutter('unable to understand error for path: %s: %s', path, err)
178
raise unknown_exc(path, extra=extra)
179
# TODO: jam 20060516 Consider re-raising the error wrapped in
180
# something like TransportError, but this loses the traceback
181
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
182
# to handle. Consider doing something like that here.
183
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
186
def _remote_path(self, relpath):
187
# XXX: It seems that ftplib does not handle Unicode paths
188
# at the same time, medusa won't handle utf8 paths So if
189
# we .encode(utf8) here (see ConnectedTransport
190
# implementation), then we get a Server failure. while
191
# if we use str(), we get a UnicodeError, and the test
192
# suite just skips testing UnicodePaths.
193
relative = str(urlutils.unescape(relpath))
194
remote_path = self._combine_paths(self._path, relative)
197
def has(self, relpath):
198
"""Does the target location exist?"""
199
# FIXME jam 20060516 We *do* ask about directories in the test suite
200
# We don't seem to in the actual codebase
201
# XXX: I assume we're never asked has(dirname) and thus I use
202
# the FTP size command and assume that if it doesn't raise,
204
abspath = self._remote_path(relpath)
207
mutter('FTP has check: %s => %s', relpath, abspath)
209
mutter("FTP has: %s", abspath)
211
except ftplib.error_perm, e:
212
if ('is a directory' in str(e).lower()):
213
mutter("FTP has dir: %s: %s", abspath, e)
215
mutter("FTP has not: %s: %s", abspath, e)
218
def get(self, relpath, decode=False, retries=0):
219
"""Get the file at the given relative path.
221
:param relpath: The relative path to the file
222
:param retries: Number of retries after temporary failures so far
225
We're meant to return a file-like object which bzr will
226
then read from. For now we do this via the magic of StringIO
228
# TODO: decode should be deprecated
230
mutter("FTP get: %s", self._remote_path(relpath))
233
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
236
except ftplib.error_perm, e:
237
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
238
except ftplib.error_temp, e:
239
if retries > _number_of_retries:
240
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
241
% self.abspath(relpath),
244
warning("FTP temporary error: %s. Retrying.", str(e))
246
return self.get(relpath, decode, retries+1)
248
if retries > _number_of_retries:
249
raise errors.TransportError("FTP control connection closed during GET %s."
250
% self.abspath(relpath),
253
warning("FTP control connection closed. Trying to reopen.")
254
time.sleep(_sleep_between_retries)
256
return self.get(relpath, decode, retries+1)
258
def put_file(self, relpath, fp, mode=None, retries=0):
259
"""Copy the file-like or string object into the location.
261
:param relpath: Location to put the contents, relative to base.
262
:param fp: File-like or string object.
263
:param retries: Number of retries after temporary failures so far
266
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
269
abspath = self._remote_path(relpath)
270
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
271
os.getpid(), random.randint(0,0x7FFFFFFF))
272
if getattr(fp, 'read', None) is None:
275
mutter("FTP put: %s", abspath)
278
f.storbinary('STOR '+tmp_abspath, fp)
279
self._rename_and_overwrite(tmp_abspath, abspath, f)
280
except (ftplib.error_temp,EOFError), e:
281
warning("Failure during ftp PUT. Deleting temporary file.")
283
f.delete(tmp_abspath)
285
warning("Failed to delete temporary file on the"
286
" server.\nFile: %s", tmp_abspath)
289
except ftplib.error_perm, e:
290
self._translate_perm_error(e, abspath, extra='could not store',
291
unknown_exc=errors.NoSuchFile)
292
except ftplib.error_temp, e:
293
if retries > _number_of_retries:
294
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
295
% self.abspath(relpath), orig_error=e)
297
warning("FTP temporary error: %s. Retrying.", str(e))
299
self.put_file(relpath, fp, mode, retries+1)
301
if retries > _number_of_retries:
302
raise errors.TransportError("FTP control connection closed during PUT %s."
303
% self.abspath(relpath), orig_error=e)
305
warning("FTP control connection closed. Trying to reopen.")
306
time.sleep(_sleep_between_retries)
308
self.put_file(relpath, fp, mode, retries+1)
310
def mkdir(self, relpath, mode=None):
311
"""Create a directory at the given path."""
312
abspath = self._remote_path(relpath)
314
mutter("FTP mkd: %s", abspath)
317
except ftplib.error_perm, e:
318
self._translate_perm_error(e, abspath,
319
unknown_exc=errors.FileExists)
321
def rmdir(self, rel_path):
322
"""Delete the directory at rel_path"""
323
abspath = self._remote_path(rel_path)
325
mutter("FTP rmd: %s", abspath)
328
except ftplib.error_perm, e:
329
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
331
def append_file(self, relpath, f, mode=None):
332
"""Append the text in the file-like object into the final
335
abspath = self._remote_path(relpath)
336
if self.has(relpath):
337
ftp = self._get_FTP()
338
result = ftp.size(abspath)
342
mutter("FTP appe to %s", abspath)
343
self._try_append(relpath, f.read(), mode)
347
def _try_append(self, relpath, text, mode=None, retries=0):
348
"""Try repeatedly to append the given text to the file at relpath.
350
This is a recursive function. On errors, it will be called until the
351
number of retries is exceeded.
354
abspath = self._remote_path(relpath)
355
mutter("FTP appe (try %d) to %s", retries, abspath)
356
ftp = self._get_FTP()
357
ftp.voidcmd("TYPE I")
358
cmd = "APPE %s" % abspath
359
conn = ftp.transfercmd(cmd)
363
self._setmode(relpath, mode)
365
except ftplib.error_perm, e:
366
self._translate_perm_error(e, abspath, extra='error appending',
367
unknown_exc=errors.NoSuchFile)
368
except ftplib.error_temp, e:
369
if retries > _number_of_retries:
370
raise errors.TransportError("FTP temporary error during APPEND %s." \
371
"Aborting." % abspath, orig_error=e)
373
warning("FTP temporary error: %s. Retrying.", str(e))
375
self._try_append(relpath, text, mode, retries+1)
377
def _setmode(self, relpath, mode):
378
"""Set permissions on a path.
380
Only set permissions if the FTP server supports the 'SITE CHMOD'
384
mutter("FTP site chmod: setting permissions to %s on %s",
385
str(mode), self._remote_path(relpath))
386
ftp = self._get_FTP()
387
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
389
except ftplib.error_perm, e:
390
# Command probably not available on this server
391
warning("FTP Could not set permissions to %s on %s. %s",
392
str(mode), self._remote_path(relpath), str(e))
394
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
395
# to copy something to another machine. And you may be able
396
# to give it its own address as the 'to' location.
397
# So implement a fancier 'copy()'
399
def rename(self, rel_from, rel_to):
400
abs_from = self._remote_path(rel_from)
401
abs_to = self._remote_path(rel_to)
402
mutter("FTP rename: %s => %s", abs_from, abs_to)
404
return self._rename(abs_from, abs_to, f)
406
def _rename(self, abs_from, abs_to, f):
408
f.rename(abs_from, abs_to)
409
except ftplib.error_perm, e:
410
self._translate_perm_error(e, abs_from,
411
': unable to rename to %r' % (abs_to))
413
def move(self, rel_from, rel_to):
414
"""Move the item at rel_from to the location at rel_to"""
415
abs_from = self._remote_path(rel_from)
416
abs_to = self._remote_path(rel_to)
418
mutter("FTP mv: %s => %s", abs_from, abs_to)
420
self._rename_and_overwrite(abs_from, abs_to, f)
421
except ftplib.error_perm, e:
422
self._translate_perm_error(e, abs_from,
423
extra='unable to rename to %r' % (rel_to,),
424
unknown_exc=errors.PathError)
426
def _rename_and_overwrite(self, abs_from, abs_to, f):
427
"""Do a fancy rename on the remote server.
429
Using the implementation provided by osutils.
431
osutils.fancy_rename(abs_from, abs_to,
432
rename_func=lambda p1, p2: self._rename(p1, p2, f),
433
unlink_func=lambda p: self._delete(p, f))
435
def delete(self, relpath):
436
"""Delete the item at relpath"""
437
abspath = self._remote_path(relpath)
439
self._delete(abspath, f)
441
def _delete(self, abspath, f):
443
mutter("FTP rm: %s", abspath)
445
except ftplib.error_perm, e:
446
self._translate_perm_error(e, abspath, 'error deleting',
447
unknown_exc=errors.NoSuchFile)
449
def external_url(self):
450
"""See bzrlib.transport.Transport.external_url."""
451
# FTP URL's are externally usable.
455
"""See Transport.listable."""
458
def list_dir(self, relpath):
459
"""See Transport.list_dir."""
460
basepath = self._remote_path(relpath)
461
mutter("FTP nlst: %s", basepath)
464
paths = f.nlst(basepath)
465
except ftplib.error_perm, e:
466
self._translate_perm_error(e, relpath, extra='error with list_dir')
467
# If FTP.nlst returns paths prefixed by relpath, strip 'em
468
if paths and paths[0].startswith(basepath):
469
entries = [path[len(basepath)+1:] for path in paths]
472
# Remove . and .. if present
473
return [urlutils.escape(entry) for entry in entries
474
if entry not in ('.', '..')]
476
def iter_files_recursive(self):
477
"""See Transport.iter_files_recursive.
479
This is cargo-culted from the SFTP transport"""
480
mutter("FTP iter_files_recursive")
481
queue = list(self.list_dir("."))
483
relpath = queue.pop(0)
484
st = self.stat(relpath)
485
if stat.S_ISDIR(st.st_mode):
486
for i, basename in enumerate(self.list_dir(relpath)):
487
queue.insert(i, relpath+"/"+basename)
491
def stat(self, relpath):
492
"""Return the stat information for a file."""
493
abspath = self._remote_path(relpath)
495
mutter("FTP stat: %s", abspath)
497
return FtpStatResult(f, abspath)
498
except ftplib.error_perm, e:
499
self._translate_perm_error(e, abspath, extra='error w/ stat')
501
def lock_read(self, relpath):
502
"""Lock the given file for shared (read) access.
503
:return: A lock object, which should be passed to Transport.unlock()
505
# The old RemoteBranch ignore lock for reading, so we will
506
# continue that tradition and return a bogus lock object.
507
class BogusLock(object):
508
def __init__(self, path):
512
return BogusLock(relpath)
514
def lock_write(self, relpath):
515
"""Lock the given file for exclusive (write) access.
516
WARNING: many transports do not support this, so trying avoid using it
518
:return: A lock object, which should be passed to Transport.unlock()
520
return self.lock_read(relpath)
523
class FtpServer(Server):
524
"""Common code for FTP server facilities."""
528
self._ftp_server = None
530
self._async_thread = None
535
"""Calculate an ftp url to this server."""
536
return 'ftp://foo:bar@localhost:%d/' % (self._port)
538
# def get_bogus_url(self):
539
# """Return a URL which cannot be connected to."""
540
# return 'ftp://127.0.0.1:1'
542
def log(self, message):
543
"""This is used by medusa.ftp_server to log connections, etc."""
544
self.logs.append(message)
546
def setUp(self, vfs_server=None):
548
raise RuntimeError('Must have medusa to run the FtpServer')
550
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
551
"FtpServer currently assumes local transport, got %s" % vfs_server
553
self._root = os.getcwdu()
554
self._ftp_server = _ftp_server(
555
authorizer=_test_authorizer(root=self._root),
557
port=0, # bind to a random port
559
logger_object=self # Use FtpServer.log() for messages
561
self._port = self._ftp_server.getsockname()[1]
562
# Don't let it loop forever, or handle an infinite number of requests.
563
# In this case it will run for 1000s, or 10000 requests
564
self._async_thread = threading.Thread(
565
target=FtpServer._asyncore_loop_ignore_EBADF,
566
kwargs={'timeout':0.1, 'count':10000})
567
self._async_thread.setDaemon(True)
568
self._async_thread.start()
571
"""See bzrlib.transport.Server.tearDown."""
572
# have asyncore release the channel
573
self._ftp_server.del_channel()
575
self._async_thread.join()
578
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
579
"""Ignore EBADF during server shutdown.
581
We close the socket to get the server to shutdown, but this causes
582
select.select() to raise EBADF.
585
asyncore.loop(*args, **kwargs)
586
# FIXME: If we reach that point, we should raise an exception
587
# explaining that the 'count' parameter in setUp is too low or
588
# testers may wonder why their test just sits there waiting for a
589
# server that is already dead. Note that if the tester waits too
590
# long under pdb the server will also die.
591
except select.error, e:
592
if e.args[0] != errno.EBADF:
598
_test_authorizer = None
602
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
605
import medusa.filesys
606
import medusa.ftp_server
612
class test_authorizer(object):
613
"""A custom Authorizer object for running the test suite.
615
The reason we cannot use dummy_authorizer, is because it sets the
616
channel to readonly, which we don't always want to do.
619
def __init__(self, root):
622
def authorize(self, channel, username, password):
623
"""Return (success, reply_string, filesystem)"""
625
return 0, 'No Medusa.', None
627
channel.persona = -1, -1
628
if username == 'anonymous':
629
channel.read_only = 1
631
channel.read_only = 0
633
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
636
class ftp_channel(medusa.ftp_server.ftp_channel):
637
"""Customized ftp channel"""
639
def log(self, message):
640
"""Redirect logging requests."""
641
mutter('_ftp_channel: %s', message)
643
def log_info(self, message, type='info'):
644
"""Redirect logging requests."""
645
mutter('_ftp_channel %s: %s', type, message)
647
def cmd_rnfr(self, line):
648
"""Prepare for renaming a file."""
649
self._renaming = line[1]
650
self.respond('350 Ready for RNTO')
651
# TODO: jam 20060516 in testing, the ftp server seems to
652
# check that the file already exists, or it sends
653
# 550 RNFR command failed
655
def cmd_rnto(self, line):
656
"""Rename a file based on the target given.
658
rnto must be called after calling rnfr.
660
if not self._renaming:
661
self.respond('503 RNFR required first.')
662
pfrom = self.filesystem.translate(self._renaming)
663
self._renaming = None
664
pto = self.filesystem.translate(line[1])
665
if os.path.exists(pto):
666
self.respond('550 RNTO failed: file exists')
669
os.rename(pfrom, pto)
670
except (IOError, OSError), e:
671
# TODO: jam 20060516 return custom responses based on
672
# why the command failed
673
# (bialix 20070418) str(e) on Python 2.5 @ Windows
674
# sometimes don't provide expected error message;
675
# so we obtain such message via os.strerror()
676
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
678
self.respond('550 RNTO failed')
679
# For a test server, we will go ahead and just die
682
self.respond('250 Rename successful.')
684
def cmd_size(self, line):
685
"""Return the size of a file
687
This is overloaded to help the test suite determine if the
688
target is a directory.
691
if not self.filesystem.isfile(filename):
692
if self.filesystem.isdir(filename):
693
self.respond('550 "%s" is a directory' % (filename,))
695
self.respond('550 "%s" is not a file' % (filename,))
697
self.respond('213 %d'
698
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
700
def cmd_mkd(self, line):
701
"""Create a directory.
703
Overloaded because default implementation does not distinguish
704
*why* it cannot make a directory.
707
self.command_not_understood(''.join(line))
711
self.filesystem.mkdir (path)
712
self.respond ('257 MKD command successful.')
713
except (IOError, OSError), e:
714
# (bialix 20070418) str(e) on Python 2.5 @ Windows
715
# sometimes don't provide expected error message;
716
# so we obtain such message via os.strerror()
717
self.respond ('550 error creating directory: %s' %
718
os.strerror(e.errno))
720
self.respond ('550 error creating directory.')
723
class ftp_server(medusa.ftp_server.ftp_server):
724
"""Customize the behavior of the Medusa ftp_server.
726
There are a few warts on the ftp_server, based on how it expects
730
ftp_channel_class = ftp_channel
732
def __init__(self, *args, **kwargs):
733
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
734
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
736
def log(self, message):
737
"""Redirect logging requests."""
738
mutter('_ftp_server: %s', message)
740
def log_info(self, message, type='info'):
741
"""Override the asyncore.log_info so we don't stipple the screen."""
742
mutter('_ftp_server %s: %s', type, message)
744
_test_authorizer = test_authorizer
745
_ftp_channel = ftp_channel
746
_ftp_server = ftp_server
751
def get_test_permutations():
752
"""Return the permutations to be used in testing."""
753
if not _setup_medusa():
754
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
757
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
