75
74
_number_of_retries = 2
76
75
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
77
class FtpTransport(Transport):
84
78
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
80
def __init__(self, base, _provided_instance=None):
87
81
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
82
assert base.startswith('ftp://') or base.startswith('aftp://')
83
super(FtpTransport, self).__init__(base)
84
self.is_active = base.startswith('aftp://')
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
90
self._FTP_instance = _provided_instance
102
92
def _get_FTP(self):
103
93
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
94
if self._FTP_instance is not None:
95
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
108
return self._FTP_instance
149
109
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
171
:param err: The error to translate into a bzr error
172
:param path: The path which had problems
173
:param extra: Extra information which can be included
174
:param unknown_exc: If None, we will just raise the original exception
175
otherwise we raise unknown_exc(path, extra=extra)
177
# ftp error numbers are very generic, like "451: Requested action aborted,
178
# local error in processing" so unfortunately we have to match by
184
extra += ': ' + str(err)
185
if ('no such file' in s
186
or 'could not open' in s
187
or 'no such dir' in s
188
or 'could not create file' in s # vsftpd
189
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
197
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
199
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
mutter('unable to understand error for path: %s: %s', path, err)
206
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
# something like TransportError, but this loses the traceback
209
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
# to handle. Consider doing something like that here.
211
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
110
raise TransportError(msg="Error setting up connection: %s"
111
% str(e), orig_error=e)
113
def should_cache(self):
114
"""Return True if the data pulled across should be cached locally.
118
def clone(self, offset=None):
119
"""Return a new FtpTransport with root at self.base + offset.
123
return FtpTransport(self.base, self._FTP_instance)
125
return FtpTransport(self.abspath(offset), self._FTP_instance)
127
def _abspath(self, relpath):
128
assert isinstance(relpath, basestring)
129
relpath = urllib.unquote(relpath)
130
if isinstance(relpath, basestring):
131
relpath_parts = relpath.split('/')
133
# TODO: Don't call this with an array - no magic interfaces
134
relpath_parts = relpath[:]
135
if len(relpath_parts) > 1:
136
if relpath_parts[0] == '':
137
raise ValueError("path %r within branch %r seems to be absolute"
138
% (relpath, self._path))
139
basepath = self._path.split('/')
140
if len(basepath) > 0 and basepath[-1] == '':
141
basepath = basepath[:-1]
142
for p in relpath_parts:
144
if len(basepath) == 0:
145
# In most filesystems, a request for the parent
146
# of root, just returns root.
149
elif p == '.' or p == '':
153
# Possibly, we could use urlparse.urljoin() here, but
154
# I'm concerned about when it chooses to strip the last
155
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
158
def abspath(self, relpath):
159
"""Return the full url to the given relative path.
160
This can be supplied with a string or a list
162
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
214
166
def has(self, relpath):
215
"""Does the target location exist?"""
216
# FIXME jam 20060516 We *do* ask about directories in the test suite
217
# We don't seem to in the actual codebase
218
# XXX: I assume we're never asked has(dirname) and thus I use
219
# the FTP size command and assume that if it doesn't raise,
221
abspath = self._remote_path(relpath)
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
223
174
f = self._get_FTP()
224
mutter('FTP has check: %s => %s', relpath, abspath)
226
mutter("FTP has: %s", abspath)
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
228
except ftplib.error_perm, e:
229
if ('is a directory' in str(e).lower()):
230
mutter("FTP has dir: %s: %s", abspath, e)
232
mutter("FTP has not: %s: %s", abspath, e)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
235
182
def get(self, relpath, decode=False, retries=0):
280
227
:param retries: Number of retries after temporary failures so far
281
228
for this operation.
283
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
230
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
286
abspath = self._remote_path(relpath)
287
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
232
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
288
233
os.getpid(), random.randint(0,0x7FFFFFFF))
290
if getattr(fp, 'read', None) is None:
291
# hand in a string IO
295
# capture the byte count; .read() may be read only so
297
class byte_counter(object):
298
def __init__(self, fp):
300
self.counted_bytes = 0
301
def read(self, count):
302
result = self.fp.read(count)
303
self.counted_bytes += len(result)
305
fp = byte_counter(fp)
234
if not hasattr(fp, 'read'):
307
mutter("FTP put: %s", abspath)
237
mutter("FTP put: %s" % self._abspath(relpath))
308
238
f = self._get_FTP()
310
240
f.storbinary('STOR '+tmp_abspath, fp)
311
self._rename_and_overwrite(tmp_abspath, abspath, f)
312
self._setmode(relpath, mode)
313
if bytes is not None:
316
return fp.counted_bytes
241
f.rename(tmp_abspath, self._abspath(relpath))
317
242
except (ftplib.error_temp,EOFError), e:
318
243
warning("Failure during ftp PUT. Deleting temporary file.")
320
245
f.delete(tmp_abspath)
322
warning("Failed to delete temporary file on the"
323
" server.\nFile: %s", tmp_abspath)
247
warning("Failed to delete temporary file on the server.\nFile: %s"
326
251
except ftplib.error_perm, e:
327
self._translate_ftp_error(e, abspath, extra='could not store',
328
unknown_exc=errors.NoSuchFile)
252
if "no such file" in str(e).lower():
253
raise NoSuchFile("Error storing %s: %s"
254
% (self.abspath(relpath), str(e)), extra=e)
256
raise FtpTransportError(orig_error=e)
329
257
except ftplib.error_temp, e:
330
258
if retries > _number_of_retries:
331
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
259
raise TransportError("FTP temporary error during PUT %s. Aborting."
332
260
% self.abspath(relpath), orig_error=e)
334
warning("FTP temporary error: %s. Retrying.", str(e))
336
self.put_file(relpath, fp, mode, retries+1)
262
warning("FTP temporary error: %s. Retrying." % str(e))
263
self._FTP_instance = None
264
self.put(relpath, fp, mode, retries+1)
338
266
if retries > _number_of_retries:
339
raise errors.TransportError("FTP control connection closed during PUT %s."
267
raise TransportError("FTP control connection closed during PUT %s."
340
268
% self.abspath(relpath), orig_error=e)
342
270
warning("FTP control connection closed. Trying to reopen.")
343
271
time.sleep(_sleep_between_retries)
345
self.put_file(relpath, fp, mode, retries+1)
272
self._FTP_instance = None
273
self.put(relpath, fp, mode, retries+1)
347
276
def mkdir(self, relpath, mode=None):
348
277
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
351
mutter("FTP mkd: %s", abspath)
354
self._setmode(relpath, mode)
355
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
357
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
def rmdir(self, rel_path):
375
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
378
mutter("FTP rmd: %s", abspath)
381
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
384
def append_file(self, relpath, f, mode=None):
279
mutter("FTP mkd: %s" % self._abspath(relpath))
282
f.mkd(self._abspath(relpath))
283
except ftplib.error_perm, e:
285
if 'File exists' in s:
286
raise FileExists(self.abspath(relpath), extra=s)
289
except ftplib.error_perm, e:
290
raise TransportError(orig_error=e)
292
def append(self, relpath, f):
385
293
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
390
if self.has(relpath):
391
ftp = self._get_FTP()
392
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
404
def _try_append(self, relpath, text, mode=None, retries=0):
405
"""Try repeatedly to append the given text to the file at relpath.
407
This is a recursive function. On errors, it will be called until the
408
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
412
mutter("FTP appe (try %d) to %s", retries, abspath)
413
ftp = self._get_FTP()
414
cmd = "APPE %s" % abspath
415
conn = ftp.transfercmd(cmd)
418
self._setmode(relpath, mode)
420
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
430
except ftplib.error_temp, e:
431
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
436
warning("FTP temporary error: %s. Retrying.", str(e))
438
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
def _setmode(self, relpath, mode):
448
"""Set permissions on a path.
450
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
466
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
# to copy something to another machine. And you may be able
468
# to give it its own address as the 'to' location.
469
# So implement a fancier 'copy()'
471
def rename(self, rel_from, rel_to):
472
abs_from = self._remote_path(rel_from)
473
abs_to = self._remote_path(rel_to)
474
mutter("FTP rename: %s => %s", abs_from, abs_to)
476
return self._rename(abs_from, abs_to, f)
478
def _rename(self, abs_from, abs_to, f):
480
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
483
': unable to rename to %r' % (abs_to))
296
raise TransportNotPossible('ftp does not support append()')
298
def copy(self, rel_from, rel_to):
299
"""Copy the item at rel_from to the location at rel_to"""
300
raise TransportNotPossible('ftp does not (yet) support copy()')
485
302
def move(self, rel_from, rel_to):
486
303
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
490
mutter("FTP mv: %s => %s", abs_from, abs_to)
305
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
306
self._abspath(rel_to)))
491
307
f = self._get_FTP()
492
self._rename_and_overwrite(abs_from, abs_to, f)
308
f.rename(self._abspath(rel_from), self._abspath(rel_to))
493
309
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
496
unknown_exc=errors.PathError)
498
def _rename_and_overwrite(self, abs_from, abs_to, f):
499
"""Do a fancy rename on the remote server.
501
Using the implementation provided by osutils.
503
osutils.fancy_rename(abs_from, abs_to,
504
rename_func=lambda p1, p2: self._rename(p1, p2, f),
505
unlink_func=lambda p: self._delete(p, f))
310
raise TransportError(orig_error=e)
507
312
def delete(self, relpath):
508
313
"""Delete the item at relpath"""
509
abspath = self._remote_path(relpath)
511
self._delete(abspath, f)
513
def _delete(self, abspath, f):
515
mutter("FTP rm: %s", abspath)
315
mutter("FTP rm: %s" % self._abspath(relpath))
317
f.delete(self._abspath(relpath))
517
318
except ftplib.error_perm, e:
518
self._translate_ftp_error(e, abspath, 'error deleting',
519
unknown_exc=errors.NoSuchFile)
521
def external_url(self):
522
"""See bzrlib.transport.Transport.external_url."""
523
# FTP URL's are externally usable.
319
raise TransportError(orig_error=e)
526
321
def listable(self):
527
322
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
