1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
49
AppendBasedFileStream,
54
from bzrlib.transport.local import LocalURLServer
60
class FtpPathError(errors.PathError):
61
"""FTP failed for path: %(path)s%(extra)s"""
64
class FtpStatResult(object):
65
def __init__(self, f, relpath):
67
self.st_size = f.size(relpath)
68
self.st_mode = stat.S_IFREG
69
except ftplib.error_perm:
73
self.st_mode = stat.S_IFDIR
78
_number_of_retries = 2
79
_sleep_between_retries = 5
81
# FIXME: there are inconsistencies in the way temporary errors are
82
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
83
# be taken to analyze the implications for write operations (read operations
84
# are safe to retry). Overall even some read operations are never
85
# retried. --vila 20070720 (Bug #127164)
86
class FtpTransport(ConnectedTransport):
87
"""This is the transport agent for ftp:// access."""
89
def __init__(self, base, _from_transport=None):
90
"""Set the base path where files will be stored."""
91
assert base.startswith('ftp://') or base.startswith('aftp://')
92
super(FtpTransport, self).__init__(base,
93
_from_transport=_from_transport)
94
self._unqualified_scheme = 'ftp'
95
if self._scheme == 'aftp':
98
self.is_active = False
101
"""Return the ftplib.FTP instance for this object."""
102
# Ensures that a connection is established
103
connection = self._get_connection()
104
if connection is None:
105
# First connection ever
106
connection, credentials = self._create_connection()
107
self._set_connection(connection, credentials)
110
def _create_connection(self, credentials=None):
111
"""Create a new connection with the provided credentials.
113
:param credentials: The credentials needed to establish the connection.
115
:return: The created connection and its associated credentials.
117
The credentials are only the password as it may have been entered
118
interactively by the user and may be different from the one provided
119
in base url at transport creation time.
121
if credentials is None:
122
password = self._password
124
password = credentials
126
mutter("Constructing FTP instance against %r" %
127
((self._host, self._port, self._user, '********',
130
connection = ftplib.FTP()
131
connection.connect(host=self._host, port=self._port)
132
if self._user and self._user != 'anonymous' and \
133
password is not None: # '' is a valid password
134
get_password = bzrlib.ui.ui_factory.get_password
135
password = get_password(prompt='FTP %(user)s@%(host)s password',
136
user=self._user, host=self._host)
137
connection.login(user=self._user, passwd=password)
138
connection.set_pasv(not self.is_active)
139
except ftplib.error_perm, e:
140
raise errors.TransportError(msg="Error setting up connection:"
141
" %s" % str(e), orig_error=e)
142
return connection, password
144
def _reconnect(self):
145
"""Create a new connection with the previously used credentials"""
146
credentials = self.get_credentials()
147
connection, credentials = self._create_connection(credentials)
148
self._set_connection(connection, credentials)
150
def _translate_perm_error(self, err, path, extra=None,
151
unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
154
:param err: The error to translate into a bzr error
155
:param path: The path which had problems
156
:param extra: Extra information which can be included
157
:param unknown_exc: If None, we will just raise the original exception
158
otherwise we raise unknown_exc(path, extra=extra)
164
extra += ': ' + str(err)
165
if ('no such file' in s
166
or 'could not open' in s
167
or 'no such dir' in s
168
or 'could not create file' in s # vsftpd
169
or 'file doesn\'t exist' in s
171
raise errors.NoSuchFile(path, extra=extra)
172
if ('file exists' in s):
173
raise errors.FileExists(path, extra=extra)
174
if ('not a directory' in s):
175
raise errors.PathError(path, extra=extra)
177
mutter('unable to understand error for path: %s: %s', path, err)
180
raise unknown_exc(path, extra=extra)
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
182
# something like TransportError, but this loses the traceback
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
184
# to handle. Consider doing something like that here.
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def _remote_path(self, relpath):
189
# XXX: It seems that ftplib does not handle Unicode paths
190
# at the same time, medusa won't handle utf8 paths So if
191
# we .encode(utf8) here (see ConnectedTransport
192
# implementation), then we get a Server failure. while
193
# if we use str(), we get a UnicodeError, and the test
194
# suite just skips testing UnicodePaths.
195
relative = str(urlutils.unescape(relpath))
196
remote_path = self._combine_paths(self._path, relative)
199
def has(self, relpath):
200
"""Does the target location exist?"""
201
# FIXME jam 20060516 We *do* ask about directories in the test suite
202
# We don't seem to in the actual codebase
203
# XXX: I assume we're never asked has(dirname) and thus I use
204
# the FTP size command and assume that if it doesn't raise,
206
abspath = self._remote_path(relpath)
209
mutter('FTP has check: %s => %s', relpath, abspath)
211
mutter("FTP has: %s", abspath)
213
except ftplib.error_perm, e:
214
if ('is a directory' in str(e).lower()):
215
mutter("FTP has dir: %s: %s", abspath, e)
217
mutter("FTP has not: %s: %s", abspath, e)
220
def get(self, relpath, decode=False, retries=0):
221
"""Get the file at the given relative path.
223
:param relpath: The relative path to the file
224
:param retries: Number of retries after temporary failures so far
227
We're meant to return a file-like object which bzr will
228
then read from. For now we do this via the magic of StringIO
230
# TODO: decode should be deprecated
232
mutter("FTP get: %s", self._remote_path(relpath))
235
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
238
except ftplib.error_perm, e:
239
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
240
except ftplib.error_temp, e:
241
if retries > _number_of_retries:
242
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
243
% self.abspath(relpath),
246
warning("FTP temporary error: %s. Retrying.", str(e))
248
return self.get(relpath, decode, retries+1)
250
if retries > _number_of_retries:
251
raise errors.TransportError("FTP control connection closed during GET %s."
252
% self.abspath(relpath),
255
warning("FTP control connection closed. Trying to reopen.")
256
time.sleep(_sleep_between_retries)
258
return self.get(relpath, decode, retries+1)
260
def put_file(self, relpath, fp, mode=None, retries=0):
261
"""Copy the file-like or string object into the location.
263
:param relpath: Location to put the contents, relative to base.
264
:param fp: File-like or string object.
265
:param retries: Number of retries after temporary failures so far
268
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
271
abspath = self._remote_path(relpath)
272
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
273
os.getpid(), random.randint(0,0x7FFFFFFF))
274
if getattr(fp, 'read', None) is None:
277
mutter("FTP put: %s", abspath)
280
f.storbinary('STOR '+tmp_abspath, fp)
281
self._rename_and_overwrite(tmp_abspath, abspath, f)
282
except (ftplib.error_temp,EOFError), e:
283
warning("Failure during ftp PUT. Deleting temporary file.")
285
f.delete(tmp_abspath)
287
warning("Failed to delete temporary file on the"
288
" server.\nFile: %s", tmp_abspath)
291
except ftplib.error_perm, e:
292
self._translate_perm_error(e, abspath, extra='could not store',
293
unknown_exc=errors.NoSuchFile)
294
except ftplib.error_temp, e:
295
if retries > _number_of_retries:
296
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
297
% self.abspath(relpath), orig_error=e)
299
warning("FTP temporary error: %s. Retrying.", str(e))
301
self.put_file(relpath, fp, mode, retries+1)
303
if retries > _number_of_retries:
304
raise errors.TransportError("FTP control connection closed during PUT %s."
305
% self.abspath(relpath), orig_error=e)
307
warning("FTP control connection closed. Trying to reopen.")
308
time.sleep(_sleep_between_retries)
310
self.put_file(relpath, fp, mode, retries+1)
312
def mkdir(self, relpath, mode=None):
313
"""Create a directory at the given path."""
314
abspath = self._remote_path(relpath)
316
mutter("FTP mkd: %s", abspath)
319
except ftplib.error_perm, e:
320
self._translate_perm_error(e, abspath,
321
unknown_exc=errors.FileExists)
323
def open_write_stream(self, relpath, mode=None):
324
"""See Transport.open_write_stream."""
325
self.put_bytes(relpath, "", mode)
326
result = AppendBasedFileStream(self, relpath)
327
_file_streams[self.abspath(relpath)] = result
330
def recommended_page_size(self):
331
"""See Transport.recommended_page_size().
333
For FTP we suggest a large page size to reduce the overhead
334
introduced by latency.
338
def rmdir(self, rel_path):
339
"""Delete the directory at rel_path"""
340
abspath = self._remote_path(rel_path)
342
mutter("FTP rmd: %s", abspath)
345
except ftplib.error_perm, e:
346
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
348
def append_file(self, relpath, f, mode=None):
349
"""Append the text in the file-like object into the final
352
abspath = self._remote_path(relpath)
353
if self.has(relpath):
354
ftp = self._get_FTP()
355
result = ftp.size(abspath)
359
mutter("FTP appe to %s", abspath)
360
self._try_append(relpath, f.read(), mode)
364
def _try_append(self, relpath, text, mode=None, retries=0):
365
"""Try repeatedly to append the given text to the file at relpath.
367
This is a recursive function. On errors, it will be called until the
368
number of retries is exceeded.
371
abspath = self._remote_path(relpath)
372
mutter("FTP appe (try %d) to %s", retries, abspath)
373
ftp = self._get_FTP()
374
ftp.voidcmd("TYPE I")
375
cmd = "APPE %s" % abspath
376
conn = ftp.transfercmd(cmd)
380
self._setmode(relpath, mode)
382
except ftplib.error_perm, e:
383
self._translate_perm_error(e, abspath, extra='error appending',
384
unknown_exc=errors.NoSuchFile)
385
except ftplib.error_temp, e:
386
if retries > _number_of_retries:
387
raise errors.TransportError("FTP temporary error during APPEND %s." \
388
"Aborting." % abspath, orig_error=e)
390
warning("FTP temporary error: %s. Retrying.", str(e))
392
self._try_append(relpath, text, mode, retries+1)
394
def _setmode(self, relpath, mode):
395
"""Set permissions on a path.
397
Only set permissions if the FTP server supports the 'SITE CHMOD'
401
mutter("FTP site chmod: setting permissions to %s on %s",
402
str(mode), self._remote_path(relpath))
403
ftp = self._get_FTP()
404
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
406
except ftplib.error_perm, e:
407
# Command probably not available on this server
408
warning("FTP Could not set permissions to %s on %s. %s",
409
str(mode), self._remote_path(relpath), str(e))
411
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
412
# to copy something to another machine. And you may be able
413
# to give it its own address as the 'to' location.
414
# So implement a fancier 'copy()'
416
def rename(self, rel_from, rel_to):
417
abs_from = self._remote_path(rel_from)
418
abs_to = self._remote_path(rel_to)
419
mutter("FTP rename: %s => %s", abs_from, abs_to)
421
return self._rename(abs_from, abs_to, f)
423
def _rename(self, abs_from, abs_to, f):
425
f.rename(abs_from, abs_to)
426
except ftplib.error_perm, e:
427
self._translate_perm_error(e, abs_from,
428
': unable to rename to %r' % (abs_to))
430
def move(self, rel_from, rel_to):
431
"""Move the item at rel_from to the location at rel_to"""
432
abs_from = self._remote_path(rel_from)
433
abs_to = self._remote_path(rel_to)
435
mutter("FTP mv: %s => %s", abs_from, abs_to)
437
self._rename_and_overwrite(abs_from, abs_to, f)
438
except ftplib.error_perm, e:
439
self._translate_perm_error(e, abs_from,
440
extra='unable to rename to %r' % (rel_to,),
441
unknown_exc=errors.PathError)
443
def _rename_and_overwrite(self, abs_from, abs_to, f):
444
"""Do a fancy rename on the remote server.
446
Using the implementation provided by osutils.
448
osutils.fancy_rename(abs_from, abs_to,
449
rename_func=lambda p1, p2: self._rename(p1, p2, f),
450
unlink_func=lambda p: self._delete(p, f))
452
def delete(self, relpath):
453
"""Delete the item at relpath"""
454
abspath = self._remote_path(relpath)
456
self._delete(abspath, f)
458
def _delete(self, abspath, f):
460
mutter("FTP rm: %s", abspath)
462
except ftplib.error_perm, e:
463
self._translate_perm_error(e, abspath, 'error deleting',
464
unknown_exc=errors.NoSuchFile)
466
def external_url(self):
467
"""See bzrlib.transport.Transport.external_url."""
468
# FTP URL's are externally usable.
472
"""See Transport.listable."""
475
def list_dir(self, relpath):
476
"""See Transport.list_dir."""
477
basepath = self._remote_path(relpath)
478
mutter("FTP nlst: %s", basepath)
481
paths = f.nlst(basepath)
482
except ftplib.error_perm, e:
483
self._translate_perm_error(e, relpath, extra='error with list_dir')
484
# If FTP.nlst returns paths prefixed by relpath, strip 'em
485
if paths and paths[0].startswith(basepath):
486
entries = [path[len(basepath)+1:] for path in paths]
489
# Remove . and .. if present
490
return [urlutils.escape(entry) for entry in entries
491
if entry not in ('.', '..')]
493
def iter_files_recursive(self):
494
"""See Transport.iter_files_recursive.
496
This is cargo-culted from the SFTP transport"""
497
mutter("FTP iter_files_recursive")
498
queue = list(self.list_dir("."))
500
relpath = queue.pop(0)
501
st = self.stat(relpath)
502
if stat.S_ISDIR(st.st_mode):
503
for i, basename in enumerate(self.list_dir(relpath)):
504
queue.insert(i, relpath+"/"+basename)
508
def stat(self, relpath):
509
"""Return the stat information for a file."""
510
abspath = self._remote_path(relpath)
512
mutter("FTP stat: %s", abspath)
514
return FtpStatResult(f, abspath)
515
except ftplib.error_perm, e:
516
self._translate_perm_error(e, abspath, extra='error w/ stat')
518
def lock_read(self, relpath):
519
"""Lock the given file for shared (read) access.
520
:return: A lock object, which should be passed to Transport.unlock()
522
# The old RemoteBranch ignore lock for reading, so we will
523
# continue that tradition and return a bogus lock object.
524
class BogusLock(object):
525
def __init__(self, path):
529
return BogusLock(relpath)
531
def lock_write(self, relpath):
532
"""Lock the given file for exclusive (write) access.
533
WARNING: many transports do not support this, so trying avoid using it
535
:return: A lock object, which should be passed to Transport.unlock()
537
return self.lock_read(relpath)
540
class FtpServer(Server):
541
"""Common code for FTP server facilities."""
545
self._ftp_server = None
547
self._async_thread = None
552
"""Calculate an ftp url to this server."""
553
return 'ftp://foo:bar@localhost:%d/' % (self._port)
555
# def get_bogus_url(self):
556
# """Return a URL which cannot be connected to."""
557
# return 'ftp://127.0.0.1:1'
559
def log(self, message):
560
"""This is used by medusa.ftp_server to log connections, etc."""
561
self.logs.append(message)
563
def setUp(self, vfs_server=None):
565
raise RuntimeError('Must have medusa to run the FtpServer')
567
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
568
"FtpServer currently assumes local transport, got %s" % vfs_server
570
self._root = os.getcwdu()
571
self._ftp_server = _ftp_server(
572
authorizer=_test_authorizer(root=self._root),
574
port=0, # bind to a random port
576
logger_object=self # Use FtpServer.log() for messages
578
self._port = self._ftp_server.getsockname()[1]
579
# Don't let it loop forever, or handle an infinite number of requests.
580
# In this case it will run for 1000s, or 10000 requests
581
self._async_thread = threading.Thread(
582
target=FtpServer._asyncore_loop_ignore_EBADF,
583
kwargs={'timeout':0.1, 'count':10000})
584
self._async_thread.setDaemon(True)
585
self._async_thread.start()
588
"""See bzrlib.transport.Server.tearDown."""
589
# have asyncore release the channel
590
self._ftp_server.del_channel()
592
self._async_thread.join()
595
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
596
"""Ignore EBADF during server shutdown.
598
We close the socket to get the server to shutdown, but this causes
599
select.select() to raise EBADF.
602
asyncore.loop(*args, **kwargs)
603
# FIXME: If we reach that point, we should raise an exception
604
# explaining that the 'count' parameter in setUp is too low or
605
# testers may wonder why their test just sits there waiting for a
606
# server that is already dead. Note that if the tester waits too
607
# long under pdb the server will also die.
608
except select.error, e:
609
if e.args[0] != errno.EBADF:
615
_test_authorizer = None
619
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
622
import medusa.filesys
623
import medusa.ftp_server
629
class test_authorizer(object):
630
"""A custom Authorizer object for running the test suite.
632
The reason we cannot use dummy_authorizer, is because it sets the
633
channel to readonly, which we don't always want to do.
636
def __init__(self, root):
639
def authorize(self, channel, username, password):
640
"""Return (success, reply_string, filesystem)"""
642
return 0, 'No Medusa.', None
644
channel.persona = -1, -1
645
if username == 'anonymous':
646
channel.read_only = 1
648
channel.read_only = 0
650
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
653
class ftp_channel(medusa.ftp_server.ftp_channel):
654
"""Customized ftp channel"""
656
def log(self, message):
657
"""Redirect logging requests."""
658
mutter('_ftp_channel: %s', message)
660
def log_info(self, message, type='info'):
661
"""Redirect logging requests."""
662
mutter('_ftp_channel %s: %s', type, message)
664
def cmd_rnfr(self, line):
665
"""Prepare for renaming a file."""
666
self._renaming = line[1]
667
self.respond('350 Ready for RNTO')
668
# TODO: jam 20060516 in testing, the ftp server seems to
669
# check that the file already exists, or it sends
670
# 550 RNFR command failed
672
def cmd_rnto(self, line):
673
"""Rename a file based on the target given.
675
rnto must be called after calling rnfr.
677
if not self._renaming:
678
self.respond('503 RNFR required first.')
679
pfrom = self.filesystem.translate(self._renaming)
680
self._renaming = None
681
pto = self.filesystem.translate(line[1])
682
if os.path.exists(pto):
683
self.respond('550 RNTO failed: file exists')
686
os.rename(pfrom, pto)
687
except (IOError, OSError), e:
688
# TODO: jam 20060516 return custom responses based on
689
# why the command failed
690
# (bialix 20070418) str(e) on Python 2.5 @ Windows
691
# sometimes don't provide expected error message;
692
# so we obtain such message via os.strerror()
693
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
695
self.respond('550 RNTO failed')
696
# For a test server, we will go ahead and just die
699
self.respond('250 Rename successful.')
701
def cmd_size(self, line):
702
"""Return the size of a file
704
This is overloaded to help the test suite determine if the
705
target is a directory.
708
if not self.filesystem.isfile(filename):
709
if self.filesystem.isdir(filename):
710
self.respond('550 "%s" is a directory' % (filename,))
712
self.respond('550 "%s" is not a file' % (filename,))
714
self.respond('213 %d'
715
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
717
def cmd_mkd(self, line):
718
"""Create a directory.
720
Overloaded because default implementation does not distinguish
721
*why* it cannot make a directory.
724
self.command_not_understood(''.join(line))
728
self.filesystem.mkdir (path)
729
self.respond ('257 MKD command successful.')
730
except (IOError, OSError), e:
731
# (bialix 20070418) str(e) on Python 2.5 @ Windows
732
# sometimes don't provide expected error message;
733
# so we obtain such message via os.strerror()
734
self.respond ('550 error creating directory: %s' %
735
os.strerror(e.errno))
737
self.respond ('550 error creating directory.')
740
class ftp_server(medusa.ftp_server.ftp_server):
741
"""Customize the behavior of the Medusa ftp_server.
743
There are a few warts on the ftp_server, based on how it expects
747
ftp_channel_class = ftp_channel
749
def __init__(self, *args, **kwargs):
750
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
751
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
753
def log(self, message):
754
"""Redirect logging requests."""
755
mutter('_ftp_server: %s', message)
757
def log_info(self, message, type='info'):
758
"""Override the asyncore.log_info so we don't stipple the screen."""
759
mutter('_ftp_server %s: %s', type, message)
761
_test_authorizer = test_authorizer
762
_ftp_channel = ftp_channel
763
_ftp_server = ftp_server
768
def get_test_permutations():
769
"""Return the permutations to be used in testing."""
770
if not _setup_medusa():
771
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
774
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
