1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
49
AppendBasedFileStream,
54
from bzrlib.transport.local import LocalURLServer
60
class FtpPathError(errors.PathError):
61
"""FTP failed for path: %(path)s%(extra)s"""
64
class FtpStatResult(object):
65
def __init__(self, f, relpath):
67
self.st_size = f.size(relpath)
68
self.st_mode = stat.S_IFREG
69
except ftplib.error_perm:
73
self.st_mode = stat.S_IFDIR
78
_number_of_retries = 2
79
_sleep_between_retries = 5
81
# FIXME: there are inconsistencies in the way temporary errors are
82
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
83
# be taken to analyze the implications for write operations (read operations
84
# are safe to retry). Overall even some read operations are never
85
# retried. --vila 20070720 (Bug #127164)
86
class FtpTransport(ConnectedTransport):
87
"""This is the transport agent for ftp:// access."""
89
def __init__(self, base, _from_transport=None):
90
"""Set the base path where files will be stored."""
91
assert base.startswith('ftp://') or base.startswith('aftp://')
92
super(FtpTransport, self).__init__(base,
93
_from_transport=_from_transport)
94
self._unqualified_scheme = 'ftp'
95
if self._scheme == 'aftp':
98
self.is_active = False
101
"""Return the ftplib.FTP instance for this object."""
102
# Ensures that a connection is established
103
connection = self._get_connection()
104
if connection is None:
105
# First connection ever
106
connection, credentials = self._create_connection()
107
self._set_connection(connection, credentials)
110
def _create_connection(self, credentials=None):
111
"""Create a new connection with the provided credentials.
113
:param credentials: The credentials needed to establish the connection.
115
:return: The created connection and its associated credentials.
117
The credentials are only the password as it may have been entered
118
interactively by the user and may be different from the one provided
119
in base url at transport creation time.
121
if credentials is None:
122
password = self._password
124
password = credentials
126
mutter("Constructing FTP instance against %r" %
127
((self._host, self._port, self._user, '********',
130
connection = ftplib.FTP()
131
connection.connect(host=self._host, port=self._port)
132
if self._user and self._user != 'anonymous' and \
133
password is None: # '' is a valid password
134
get_password = bzrlib.ui.ui_factory.get_password
135
password = get_password(prompt='FTP %(user)s@%(host)s password',
136
user=self._user, host=self._host)
137
connection.login(user=self._user, passwd=password)
138
connection.set_pasv(not self.is_active)
139
except ftplib.error_perm, e:
140
raise errors.TransportError(msg="Error setting up connection:"
141
" %s" % str(e), orig_error=e)
142
return connection, password
144
def _reconnect(self):
145
"""Create a new connection with the previously used credentials"""
146
credentials = self.get_credentials()
147
connection, credentials = self._create_connection(credentials)
148
self._set_connection(connection, credentials)
150
def _translate_perm_error(self, err, path, extra=None,
151
unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
154
:param err: The error to translate into a bzr error
155
:param path: The path which had problems
156
:param extra: Extra information which can be included
157
:param unknown_exc: If None, we will just raise the original exception
158
otherwise we raise unknown_exc(path, extra=extra)
164
extra += ': ' + str(err)
165
if ('no such file' in s
166
or 'could not open' in s
167
or 'no such dir' in s
168
or 'could not create file' in s # vsftpd
169
or 'file doesn\'t exist' in s
171
raise errors.NoSuchFile(path, extra=extra)
172
if ('file exists' in s):
173
raise errors.FileExists(path, extra=extra)
174
if ('not a directory' in s):
175
raise errors.PathError(path, extra=extra)
177
mutter('unable to understand error for path: %s: %s', path, err)
180
raise unknown_exc(path, extra=extra)
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
182
# something like TransportError, but this loses the traceback
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
184
# to handle. Consider doing something like that here.
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def _remote_path(self, relpath):
189
# XXX: It seems that ftplib does not handle Unicode paths
190
# at the same time, medusa won't handle utf8 paths So if
191
# we .encode(utf8) here (see ConnectedTransport
192
# implementation), then we get a Server failure. while
193
# if we use str(), we get a UnicodeError, and the test
194
# suite just skips testing UnicodePaths.
195
relative = str(urlutils.unescape(relpath))
196
remote_path = self._combine_paths(self._path, relative)
199
def has(self, relpath):
200
"""Does the target location exist?"""
201
# FIXME jam 20060516 We *do* ask about directories in the test suite
202
# We don't seem to in the actual codebase
203
# XXX: I assume we're never asked has(dirname) and thus I use
204
# the FTP size command and assume that if it doesn't raise,
206
abspath = self._remote_path(relpath)
209
mutter('FTP has check: %s => %s', relpath, abspath)
211
mutter("FTP has: %s", abspath)
213
except ftplib.error_perm, e:
214
if ('is a directory' in str(e).lower()):
215
mutter("FTP has dir: %s: %s", abspath, e)
217
mutter("FTP has not: %s: %s", abspath, e)
220
def get(self, relpath, decode=False, retries=0):
221
"""Get the file at the given relative path.
223
:param relpath: The relative path to the file
224
:param retries: Number of retries after temporary failures so far
227
We're meant to return a file-like object which bzr will
228
then read from. For now we do this via the magic of StringIO
230
# TODO: decode should be deprecated
232
mutter("FTP get: %s", self._remote_path(relpath))
235
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
238
except ftplib.error_perm, e:
239
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
240
except ftplib.error_temp, e:
241
if retries > _number_of_retries:
242
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
243
% self.abspath(relpath),
246
warning("FTP temporary error: %s. Retrying.", str(e))
248
return self.get(relpath, decode, retries+1)
250
if retries > _number_of_retries:
251
raise errors.TransportError("FTP control connection closed during GET %s."
252
% self.abspath(relpath),
255
warning("FTP control connection closed. Trying to reopen.")
256
time.sleep(_sleep_between_retries)
258
return self.get(relpath, decode, retries+1)
260
def put_file(self, relpath, fp, mode=None, retries=0):
261
"""Copy the file-like or string object into the location.
263
:param relpath: Location to put the contents, relative to base.
264
:param fp: File-like or string object.
265
:param retries: Number of retries after temporary failures so far
268
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
271
abspath = self._remote_path(relpath)
272
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
273
os.getpid(), random.randint(0,0x7FFFFFFF))
275
if getattr(fp, 'read', None) is None:
276
# hand in a string IO
280
# capture the byte count; .read() may be read only so
282
class byte_counter(object):
283
def __init__(self, fp):
285
self.counted_bytes = 0
286
def read(self, count):
287
result = self.fp.read(count)
288
self.counted_bytes += len(result)
290
fp = byte_counter(fp)
292
mutter("FTP put: %s", abspath)
295
f.storbinary('STOR '+tmp_abspath, fp)
296
self._rename_and_overwrite(tmp_abspath, abspath, f)
297
if bytes is not None:
300
return fp.counted_bytes
301
except (ftplib.error_temp,EOFError), e:
302
warning("Failure during ftp PUT. Deleting temporary file.")
304
f.delete(tmp_abspath)
306
warning("Failed to delete temporary file on the"
307
" server.\nFile: %s", tmp_abspath)
310
except ftplib.error_perm, e:
311
self._translate_perm_error(e, abspath, extra='could not store',
312
unknown_exc=errors.NoSuchFile)
313
except ftplib.error_temp, e:
314
if retries > _number_of_retries:
315
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
316
% self.abspath(relpath), orig_error=e)
318
warning("FTP temporary error: %s. Retrying.", str(e))
320
self.put_file(relpath, fp, mode, retries+1)
322
if retries > _number_of_retries:
323
raise errors.TransportError("FTP control connection closed during PUT %s."
324
% self.abspath(relpath), orig_error=e)
326
warning("FTP control connection closed. Trying to reopen.")
327
time.sleep(_sleep_between_retries)
329
self.put_file(relpath, fp, mode, retries+1)
331
def mkdir(self, relpath, mode=None):
332
"""Create a directory at the given path."""
333
abspath = self._remote_path(relpath)
335
mutter("FTP mkd: %s", abspath)
338
except ftplib.error_perm, e:
339
self._translate_perm_error(e, abspath,
340
unknown_exc=errors.FileExists)
342
def open_write_stream(self, relpath, mode=None):
343
"""See Transport.open_write_stream."""
344
self.put_bytes(relpath, "", mode)
345
result = AppendBasedFileStream(self, relpath)
346
_file_streams[self.abspath(relpath)] = result
349
def recommended_page_size(self):
350
"""See Transport.recommended_page_size().
352
For FTP we suggest a large page size to reduce the overhead
353
introduced by latency.
357
def rmdir(self, rel_path):
358
"""Delete the directory at rel_path"""
359
abspath = self._remote_path(rel_path)
361
mutter("FTP rmd: %s", abspath)
364
except ftplib.error_perm, e:
365
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
367
def append_file(self, relpath, f, mode=None):
368
"""Append the text in the file-like object into the final
371
abspath = self._remote_path(relpath)
372
if self.has(relpath):
373
ftp = self._get_FTP()
374
result = ftp.size(abspath)
378
mutter("FTP appe to %s", abspath)
379
self._try_append(relpath, f.read(), mode)
383
def _try_append(self, relpath, text, mode=None, retries=0):
384
"""Try repeatedly to append the given text to the file at relpath.
386
This is a recursive function. On errors, it will be called until the
387
number of retries is exceeded.
390
abspath = self._remote_path(relpath)
391
mutter("FTP appe (try %d) to %s", retries, abspath)
392
ftp = self._get_FTP()
393
ftp.voidcmd("TYPE I")
394
cmd = "APPE %s" % abspath
395
conn = ftp.transfercmd(cmd)
399
self._setmode(relpath, mode)
401
except ftplib.error_perm, e:
402
self._translate_perm_error(e, abspath, extra='error appending',
403
unknown_exc=errors.NoSuchFile)
404
except ftplib.error_temp, e:
405
if retries > _number_of_retries:
406
raise errors.TransportError("FTP temporary error during APPEND %s." \
407
"Aborting." % abspath, orig_error=e)
409
warning("FTP temporary error: %s. Retrying.", str(e))
411
self._try_append(relpath, text, mode, retries+1)
413
def _setmode(self, relpath, mode):
414
"""Set permissions on a path.
416
Only set permissions if the FTP server supports the 'SITE CHMOD'
420
mutter("FTP site chmod: setting permissions to %s on %s",
421
str(mode), self._remote_path(relpath))
422
ftp = self._get_FTP()
423
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
425
except ftplib.error_perm, e:
426
# Command probably not available on this server
427
warning("FTP Could not set permissions to %s on %s. %s",
428
str(mode), self._remote_path(relpath), str(e))
430
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
431
# to copy something to another machine. And you may be able
432
# to give it its own address as the 'to' location.
433
# So implement a fancier 'copy()'
435
def rename(self, rel_from, rel_to):
436
abs_from = self._remote_path(rel_from)
437
abs_to = self._remote_path(rel_to)
438
mutter("FTP rename: %s => %s", abs_from, abs_to)
440
return self._rename(abs_from, abs_to, f)
442
def _rename(self, abs_from, abs_to, f):
444
f.rename(abs_from, abs_to)
445
except ftplib.error_perm, e:
446
self._translate_perm_error(e, abs_from,
447
': unable to rename to %r' % (abs_to))
449
def move(self, rel_from, rel_to):
450
"""Move the item at rel_from to the location at rel_to"""
451
abs_from = self._remote_path(rel_from)
452
abs_to = self._remote_path(rel_to)
454
mutter("FTP mv: %s => %s", abs_from, abs_to)
456
self._rename_and_overwrite(abs_from, abs_to, f)
457
except ftplib.error_perm, e:
458
self._translate_perm_error(e, abs_from,
459
extra='unable to rename to %r' % (rel_to,),
460
unknown_exc=errors.PathError)
462
def _rename_and_overwrite(self, abs_from, abs_to, f):
463
"""Do a fancy rename on the remote server.
465
Using the implementation provided by osutils.
467
osutils.fancy_rename(abs_from, abs_to,
468
rename_func=lambda p1, p2: self._rename(p1, p2, f),
469
unlink_func=lambda p: self._delete(p, f))
471
def delete(self, relpath):
472
"""Delete the item at relpath"""
473
abspath = self._remote_path(relpath)
475
self._delete(abspath, f)
477
def _delete(self, abspath, f):
479
mutter("FTP rm: %s", abspath)
481
except ftplib.error_perm, e:
482
self._translate_perm_error(e, abspath, 'error deleting',
483
unknown_exc=errors.NoSuchFile)
485
def external_url(self):
486
"""See bzrlib.transport.Transport.external_url."""
487
# FTP URL's are externally usable.
491
"""See Transport.listable."""
494
def list_dir(self, relpath):
495
"""See Transport.list_dir."""
496
basepath = self._remote_path(relpath)
497
mutter("FTP nlst: %s", basepath)
500
paths = f.nlst(basepath)
501
except ftplib.error_perm, e:
502
self._translate_perm_error(e, relpath, extra='error with list_dir')
503
# If FTP.nlst returns paths prefixed by relpath, strip 'em
504
if paths and paths[0].startswith(basepath):
505
entries = [path[len(basepath)+1:] for path in paths]
508
# Remove . and .. if present
509
return [urlutils.escape(entry) for entry in entries
510
if entry not in ('.', '..')]
512
def iter_files_recursive(self):
513
"""See Transport.iter_files_recursive.
515
This is cargo-culted from the SFTP transport"""
516
mutter("FTP iter_files_recursive")
517
queue = list(self.list_dir("."))
519
relpath = queue.pop(0)
520
st = self.stat(relpath)
521
if stat.S_ISDIR(st.st_mode):
522
for i, basename in enumerate(self.list_dir(relpath)):
523
queue.insert(i, relpath+"/"+basename)
527
def stat(self, relpath):
528
"""Return the stat information for a file."""
529
abspath = self._remote_path(relpath)
531
mutter("FTP stat: %s", abspath)
533
return FtpStatResult(f, abspath)
534
except ftplib.error_perm, e:
535
self._translate_perm_error(e, abspath, extra='error w/ stat')
537
def lock_read(self, relpath):
538
"""Lock the given file for shared (read) access.
539
:return: A lock object, which should be passed to Transport.unlock()
541
# The old RemoteBranch ignore lock for reading, so we will
542
# continue that tradition and return a bogus lock object.
543
class BogusLock(object):
544
def __init__(self, path):
548
return BogusLock(relpath)
550
def lock_write(self, relpath):
551
"""Lock the given file for exclusive (write) access.
552
WARNING: many transports do not support this, so trying avoid using it
554
:return: A lock object, which should be passed to Transport.unlock()
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for FTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 1000s, or 10000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':10000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
self._ftp_server.close()
610
self._async_thread.join()
613
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
614
"""Ignore EBADF during server shutdown.
616
We close the socket to get the server to shutdown, but this causes
617
select.select() to raise EBADF.
620
asyncore.loop(*args, **kwargs)
621
# FIXME: If we reach that point, we should raise an exception
622
# explaining that the 'count' parameter in setUp is too low or
623
# testers may wonder why their test just sits there waiting for a
624
# server that is already dead. Note that if the tester waits too
625
# long under pdb the server will also die.
626
except select.error, e:
627
if e.args[0] != errno.EBADF:
633
_test_authorizer = None
637
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
640
import medusa.filesys
641
import medusa.ftp_server
647
class test_authorizer(object):
648
"""A custom Authorizer object for running the test suite.
650
The reason we cannot use dummy_authorizer, is because it sets the
651
channel to readonly, which we don't always want to do.
654
def __init__(self, root):
656
# If secured_user is set secured_password will be checked
657
self.secured_user = None
658
self.secured_password = None
660
def authorize(self, channel, username, password):
661
"""Return (success, reply_string, filesystem)"""
663
return 0, 'No Medusa.', None
665
channel.persona = -1, -1
666
if username == 'anonymous':
667
channel.read_only = 1
669
channel.read_only = 0
671
# Check secured_user if set
672
if (self.secured_user is not None
673
and username == self.secured_user
674
and password != self.secured_password):
675
return 0, 'Password invalid.', None
677
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
680
class ftp_channel(medusa.ftp_server.ftp_channel):
681
"""Customized ftp channel"""
683
def log(self, message):
684
"""Redirect logging requests."""
685
mutter('_ftp_channel: %s', message)
687
def log_info(self, message, type='info'):
688
"""Redirect logging requests."""
689
mutter('_ftp_channel %s: %s', type, message)
691
def cmd_rnfr(self, line):
692
"""Prepare for renaming a file."""
693
self._renaming = line[1]
694
self.respond('350 Ready for RNTO')
695
# TODO: jam 20060516 in testing, the ftp server seems to
696
# check that the file already exists, or it sends
697
# 550 RNFR command failed
699
def cmd_rnto(self, line):
700
"""Rename a file based on the target given.
702
rnto must be called after calling rnfr.
704
if not self._renaming:
705
self.respond('503 RNFR required first.')
706
pfrom = self.filesystem.translate(self._renaming)
707
self._renaming = None
708
pto = self.filesystem.translate(line[1])
709
if os.path.exists(pto):
710
self.respond('550 RNTO failed: file exists')
713
os.rename(pfrom, pto)
714
except (IOError, OSError), e:
715
# TODO: jam 20060516 return custom responses based on
716
# why the command failed
717
# (bialix 20070418) str(e) on Python 2.5 @ Windows
718
# sometimes don't provide expected error message;
719
# so we obtain such message via os.strerror()
720
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
722
self.respond('550 RNTO failed')
723
# For a test server, we will go ahead and just die
726
self.respond('250 Rename successful.')
728
def cmd_size(self, line):
729
"""Return the size of a file
731
This is overloaded to help the test suite determine if the
732
target is a directory.
735
if not self.filesystem.isfile(filename):
736
if self.filesystem.isdir(filename):
737
self.respond('550 "%s" is a directory' % (filename,))
739
self.respond('550 "%s" is not a file' % (filename,))
741
self.respond('213 %d'
742
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
744
def cmd_mkd(self, line):
745
"""Create a directory.
747
Overloaded because default implementation does not distinguish
748
*why* it cannot make a directory.
751
self.command_not_understood(''.join(line))
755
self.filesystem.mkdir (path)
756
self.respond ('257 MKD command successful.')
757
except (IOError, OSError), e:
758
# (bialix 20070418) str(e) on Python 2.5 @ Windows
759
# sometimes don't provide expected error message;
760
# so we obtain such message via os.strerror()
761
self.respond ('550 error creating directory: %s' %
762
os.strerror(e.errno))
764
self.respond ('550 error creating directory.')
767
class ftp_server(medusa.ftp_server.ftp_server):
768
"""Customize the behavior of the Medusa ftp_server.
770
There are a few warts on the ftp_server, based on how it expects
774
ftp_channel_class = ftp_channel
776
def __init__(self, *args, **kwargs):
777
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
778
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
780
def log(self, message):
781
"""Redirect logging requests."""
782
mutter('_ftp_server: %s', message)
784
def log_info(self, message, type='info'):
785
"""Override the asyncore.log_info so we don't stipple the screen."""
786
mutter('_ftp_server %s: %s', type, message)
788
_test_authorizer = test_authorizer
789
_ftp_channel = ftp_channel
790
_ftp_server = ftp_server
795
def get_test_permutations():
796
"""Return the permutations to be used in testing."""
797
if not _setup_medusa():
798
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
801
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
