1
# Copyright (C) 2005-2012, 2016 Canonical Ltd
1
# Copyright (C) 2005, 2006 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
"""Transport for the local filesystem.
19
This is a fairly thin wrapper on regular file IO.
22
from __future__ import absolute_import
19
This is a fairly thin wrapper on regular file IO."""
25
from stat import ST_MODE, S_ISDIR, S_IMODE
28
from ..lazy_import import lazy_import
29
lazy_import(globals(), """
38
from breezy.transport import LateReadError
41
from .. import transport
44
_append_flags = os.O_CREAT | os.O_APPEND | os.O_WRONLY | osutils.O_BINARY | osutils.O_NOINHERIT
45
_put_non_atomic_flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | osutils.O_BINARY | osutils.O_NOINHERIT
48
class LocalTransport(transport.Transport):
24
from stat import ST_MODE, S_ISDIR, ST_SIZE
28
from bzrlib.trace import mutter
29
from bzrlib.transport import Transport, Server
30
from bzrlib.osutils import (abspath, realpath, normpath, pathjoin, rename,
34
class LocalTransport(Transport):
49
35
"""This is the transport agent for local filesystem access."""
51
37
def __init__(self, base):
52
38
"""Set the base path where files will be stored."""
53
if not base.startswith('file://'):
54
raise AssertionError("not a file:// url: %r" % base)
39
if base.startswith('file://'):
40
base = base[len('file://'):]
41
# realpath is incompatible with symlinks. When we traverse
42
# up we might be able to normpath stuff. RBC 20051003
43
base = normpath(abspath(base))
55
44
if base[-1] != '/':
58
# Special case : windows has no "root", but does have
59
# multiple lettered drives inside it. #240910
60
if sys.platform == 'win32' and base == 'file:///':
63
super(LocalTransport, self).__init__(base)
66
46
super(LocalTransport, self).__init__(base)
67
self._local_base = urlutils.local_path_from_url(base)
68
if self._local_base[-1] != '/':
69
self._local_base = self._local_base + '/'
48
def should_cache(self):
71
51
def clone(self, offset=None):
72
52
"""Return a new LocalTransport with root at self.base + offset
73
Because the local filesystem does not require a connection,
53
Because the local filesystem does not require a connection,
74
54
we can just return a new object.
77
57
return LocalTransport(self.base)
79
abspath = self.abspath(offset)
80
if abspath == 'file://':
81
# fix upwalk for UNC path
82
# when clone from //HOST/path updir recursively
83
# we should stop at least at //HOST part
85
return LocalTransport(abspath)
87
def _abspath(self, relative_reference):
88
"""Return a path for use in os calls.
90
Several assumptions are made:
91
- relative_reference does not contain '..'
92
- relative_reference is url escaped.
94
if relative_reference in ('.', ''):
95
# _local_base normally has a trailing slash; strip it so that stat
96
# on a transport pointing to a symlink reads the link not the
97
# referent but be careful of / and c:\
98
return osutils.split(self._local_base)[0]
99
return self._local_base + urlutils.unescape(relative_reference)
59
return LocalTransport(self.abspath(offset))
101
61
def abspath(self, relpath):
102
62
"""Return the full url to the given relative URL."""
103
# TODO: url escape the result. RBC 20060523.
104
# jam 20060426 Using normpath on the real path, because that ensures
105
# proper handling of stuff like
106
path = osutils.normpath(osutils.pathjoin(
107
self._local_base, urlutils.unescape(relpath)))
108
# on windows, our _local_base may or may not have a drive specified
109
# (ie, it may be "/" or "c:/foo").
110
# If 'relpath' is '/' we *always* get back an abspath without
111
# the drive letter - but if our transport already has a drive letter,
112
# we want our abspaths to have a drive letter too - so handle that
114
if (sys.platform == "win32" and self._local_base[1:2] == ":"
116
path = self._local_base[:3]
118
return urlutils.local_path_to_url(path)
120
def local_abspath(self, relpath):
121
"""Transform the given relative path URL into the actual path on disk
123
This function only exists for the LocalTransport, since it is
124
the only one that has direct local access.
125
This is mostly for stuff like WorkingTree which needs to know
126
the local working directory. The returned path will always contain
127
forward slashes as the path separator, regardless of the platform.
129
This function is quite expensive: it calls realpath which resolves
132
absurl = self.abspath(relpath)
133
# mutter(u'relpath %s => base: %s, absurl %s', relpath, self.base, absurl)
134
return urlutils.local_path_from_url(absurl)
63
assert isinstance(relpath, basestring), (type(relpath), relpath)
64
result = normpath(pathjoin(self.base, urllib.unquote(relpath)))
65
#if result[-1] != '/':
136
69
def relpath(self, abspath):
137
70
"""Return the local path portion from a given absolute path.
72
from bzrlib.osutils import relpath, strip_trailing_slash
139
73
if abspath is None:
142
return urlutils.file_relpath(self.base, abspath)
76
return relpath(strip_trailing_slash(self.base),
77
strip_trailing_slash(abspath))
144
79
def has(self, relpath):
145
return os.access(self._abspath(relpath), os.F_OK)
80
return os.access(self.abspath(relpath), os.F_OK)
147
82
def get(self, relpath):
148
83
"""Get the file at the given relative path.
150
85
:param relpath: The relative path to the file
152
canonical_url = self.abspath(relpath)
153
if canonical_url in transport._file_streams:
154
transport._file_streams[canonical_url].flush()
156
path = self._abspath(relpath)
157
return osutils.open_file(path, 'rb')
158
except (IOError, OSError) as e:
159
if e.errno == errno.EISDIR:
160
return LateReadError(relpath)
161
self._translate_error(e, path)
163
def put_file(self, relpath, f, mode=None):
164
"""Copy the file-like object into the location.
166
:param relpath: Location to put the contents, relative to base.
167
:param f: File-like object.
168
:param mode: The mode for the newly created file,
169
None means just use the default
174
path = self._abspath(relpath)
175
osutils.check_legal_path(path)
176
fp = atomicfile.AtomicFile(path, 'wb', new_mode=mode)
177
except (IOError, OSError) as e:
178
self._translate_error(e, path)
180
length = self._pump(f, fp)
186
def put_bytes(self, relpath, raw_bytes, mode=None):
187
"""Copy the string into the location.
189
:param relpath: Location to put the contents, relative to base.
190
:param raw_bytes: String
192
if not isinstance(raw_bytes, bytes):
194
'raw_bytes must be bytes, not %s' % type(raw_bytes))
197
path = self._abspath(relpath)
198
osutils.check_legal_path(path)
199
fp = atomicfile.AtomicFile(path, 'wb', new_mode=mode)
200
except (IOError, OSError) as e:
201
self._translate_error(e, path)
209
def _put_non_atomic_helper(self, relpath, writer,
211
create_parent_dir=False,
213
"""Common functionality information for the put_*_non_atomic.
215
This tracks all the create_parent_dir stuff.
217
:param relpath: the path we are putting to.
218
:param writer: A function that takes an os level file descriptor
219
and writes whatever data it needs to write there.
220
:param mode: The final file mode.
221
:param create_parent_dir: Should we be creating the parent directory
224
abspath = self._abspath(relpath)
226
# os.open() will automatically use the umask
231
fd = os.open(abspath, _put_non_atomic_flags, local_mode)
232
except (IOError, OSError) as e:
233
# We couldn't create the file, maybe we need to create
234
# the parent directory, and try again
235
if (not create_parent_dir
236
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
237
self._translate_error(e, relpath)
238
parent_dir = os.path.dirname(abspath)
240
self._translate_error(e, relpath)
241
self._mkdir(parent_dir, mode=dir_mode)
242
# We created the parent directory, lets try to open the
245
fd = os.open(abspath, _put_non_atomic_flags, local_mode)
246
except (IOError, OSError) as e:
247
self._translate_error(e, relpath)
250
if mode is not None and mode != S_IMODE(st.st_mode):
251
# Because of umask, we may still need to chmod the file.
252
# But in the general case, we won't have to
253
osutils.chmod_if_possible(abspath, mode)
258
def put_file_non_atomic(self, relpath, f, mode=None,
259
create_parent_dir=False,
261
"""Copy the file-like object into the target location.
263
This function is not strictly safe to use. It is only meant to
264
be used when you already know that the target does not exist.
265
It is not safe, because it will open and truncate the remote
266
file. So there may be a time when the file has invalid contents.
268
:param relpath: The remote location to put the contents.
269
:param f: File-like object.
270
:param mode: Possible access permissions for new file.
271
None means do not set remote permissions.
272
:param create_parent_dir: If we cannot create the target file because
273
the parent directory does not exist, go ahead and
274
create it, and then try again.
277
self._pump_to_fd(f, fd)
278
self._put_non_atomic_helper(relpath, writer, mode=mode,
279
create_parent_dir=create_parent_dir,
282
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
283
create_parent_dir=False, dir_mode=None):
287
self._put_non_atomic_helper(relpath, writer, mode=mode,
288
create_parent_dir=create_parent_dir,
88
path = self.abspath(relpath)
89
return open(path, 'rb')
90
except (IOError, OSError),e:
91
self._translate_error(e, path)
93
def put(self, relpath, f, mode=None):
94
"""Copy the file-like or string object into the location.
96
:param relpath: Location to put the contents, relative to base.
97
:param f: File-like or string object.
99
from bzrlib.atomicfile import AtomicFile
103
path = self.abspath(relpath)
104
check_legal_path(path)
105
fp = AtomicFile(path, 'wb', new_mode=mode)
106
except (IOError, OSError),e:
107
self._translate_error(e, path)
291
114
def iter_files_recursive(self):
292
115
"""Iter the relative paths of files in the transports sub-tree."""
296
119
st = self.stat(relpath)
297
120
if S_ISDIR(st[ST_MODE]):
298
121
for i, basename in enumerate(self.list_dir(relpath)):
299
queue.insert(i, relpath + '/' + basename)
122
queue.insert(i, relpath+'/'+basename)
303
def _mkdir(self, abspath, mode=None):
304
"""Create a real directory, filtering through mode"""
306
# os.mkdir() will filter through umask
311
os.mkdir(abspath, local_mode)
312
except (IOError, OSError) as e:
313
self._translate_error(e, abspath)
316
osutils.chmod_if_possible(abspath, mode)
317
except (IOError, OSError) as e:
318
self._translate_error(e, abspath)
320
126
def mkdir(self, relpath, mode=None):
321
127
"""Create a directory at the given path."""
322
self._mkdir(self._abspath(relpath), mode=mode)
324
def open_write_stream(self, relpath, mode=None):
325
"""See Transport.open_write_stream."""
326
abspath = self._abspath(relpath)
328
handle = osutils.open_file(abspath, 'wb')
329
except (IOError, OSError) as e:
330
self._translate_error(e, abspath)
333
self._check_mode_and_size(abspath, handle.fileno(), mode)
334
transport._file_streams[self.abspath(relpath)] = handle
335
return transport.FileFileStream(self, relpath, handle)
337
def _get_append_file(self, relpath, mode=None):
338
"""Call os.open() for the given relpath"""
339
file_abspath = self._abspath(relpath)
341
# os.open() will automatically use the umask
346
return file_abspath, os.open(file_abspath, _append_flags, local_mode)
347
except (IOError, OSError) as e:
130
path = self.abspath(relpath)
134
except (IOError, OSError),e:
135
self._translate_error(e, path)
137
def append(self, relpath, f, mode=None):
138
"""Append the text in the file-like object into the final
142
fp = open(self.abspath(relpath), 'ab')
144
os.chmod(self.abspath(relpath), mode)
145
except (IOError, OSError),e:
348
146
self._translate_error(e, relpath)
350
def _check_mode_and_size(self, file_abspath, fd, mode=None):
351
"""Check the mode of the file, and return the current size"""
353
if mode is not None and mode != S_IMODE(st.st_mode):
354
# Because of umask, we may still need to chmod the file.
355
# But in the general case, we won't have to
356
osutils.chmod_if_possible(file_abspath, mode)
359
def append_file(self, relpath, f, mode=None):
360
"""Append the text in the file-like object into the final location."""
361
file_abspath, fd = self._get_append_file(relpath, mode=mode)
363
result = self._check_mode_and_size(file_abspath, fd, mode=mode)
364
self._pump_to_fd(f, fd)
369
def append_bytes(self, relpath, bytes, mode=None):
370
"""Append the text in the string into the final location."""
371
file_abspath, fd = self._get_append_file(relpath, mode=mode)
373
result = self._check_mode_and_size(file_abspath, fd, mode=mode)
380
def _pump_to_fd(self, fromfile, to_fd):
381
"""Copy contents of one file to another."""
384
b = fromfile.read(BUFSIZE)
147
# win32 workaround (tell on an unwritten file returns 0)
389
153
def copy(self, rel_from, rel_to):
390
154
"""Copy the item at rel_from to the location at rel_to"""
391
path_from = self._abspath(rel_from)
392
path_to = self._abspath(rel_to)
156
path_from = self.abspath(rel_from)
157
path_to = self.abspath(rel_to)
394
159
shutil.copy(path_from, path_to)
395
except (IOError, OSError) as e:
160
except (IOError, OSError),e:
396
161
# TODO: What about path_to?
397
162
self._translate_error(e, path_from)
399
164
def rename(self, rel_from, rel_to):
400
path_from = self._abspath(rel_from)
401
path_to = self._abspath(rel_to)
165
path_from = self.abspath(rel_from)
403
# *don't* call breezy.osutils.rename, because we want to
404
# detect conflicting names on rename, and osutils.rename tries to
405
# mask cross-platform differences there
406
os.rename(path_from, path_to)
407
except (IOError, OSError) as e:
167
# *don't* call bzrlib.osutils.rename, because we want to
168
# detect errors on rename
169
os.rename(path_from, self.abspath(rel_to))
170
except (IOError, OSError),e:
408
171
# TODO: What about path_to?
409
172
self._translate_error(e, path_from)
411
174
def move(self, rel_from, rel_to):
412
175
"""Move the item at rel_from to the location at rel_to"""
413
path_from = self._abspath(rel_from)
414
path_to = self._abspath(rel_to)
176
path_from = self.abspath(rel_from)
177
path_to = self.abspath(rel_to)
417
180
# this version will delete the destination if necessary
418
osutils.rename(path_from, path_to)
419
except (IOError, OSError) as e:
181
rename(path_from, path_to)
182
except (IOError, OSError),e:
420
183
# TODO: What about path_to?
421
184
self._translate_error(e, path_from)
555
class EmulatedWin32LocalTransport(LocalTransport):
556
"""Special transport for testing Win32 [UNC] paths on non-windows"""
558
def __init__(self, base):
561
super(LocalTransport, self).__init__(base)
562
self._local_base = urlutils._win32_local_path_from_url(base)
564
def abspath(self, relpath):
565
path = osutils._win32_normpath(osutils.pathjoin(
566
self._local_base, urlutils.unescape(relpath)))
567
return urlutils._win32_local_path_to_url(path)
569
def clone(self, offset=None):
570
"""Return a new LocalTransport with root at self.base + offset
571
Because the local filesystem does not require a connection,
572
we can just return a new object.
575
return EmulatedWin32LocalTransport(self.base)
577
abspath = self.abspath(offset)
578
if abspath == 'file://':
579
# fix upwalk for UNC path
580
# when clone from //HOST/path updir recursively
581
# we should stop at least at //HOST part
583
return EmulatedWin32LocalTransport(abspath)
287
class ScratchTransport(LocalTransport):
288
"""A transport that works in a temporary dir and cleans up after itself.
290
The dir only exists for the lifetime of the Python object.
291
Obviously you should not put anything precious in it.
294
def __init__(self, base=None):
296
base = tempfile.mkdtemp()
297
super(ScratchTransport, self).__init__(base)
300
shutil.rmtree(self.base, ignore_errors=True)
301
mutter("%r destroyed" % self)
304
class LocalRelpathServer(Server):
305
"""A pretend server for local transports, using relpaths."""
308
"""See Transport.Server.get_url."""
312
class LocalAbspathServer(Server):
313
"""A pretend server for local transports, using absolute paths."""
316
"""See Transport.Server.get_url."""
317
return os.path.abspath("")
320
class LocalURLServer(Server):
321
"""A pretend server for local transports, using file:// urls."""
324
"""See Transport.Server.get_url."""
325
# FIXME: \ to / on windows
326
return "file://%s" % os.path.abspath("")
586
329
def get_test_permutations():
587
330
"""Return the permutations to be used in testing."""
588
from ..tests import test_server
589
return [(LocalTransport, test_server.LocalURLServer), ]
331
return [(LocalTransport, LocalRelpathServer),
332
(LocalTransport, LocalAbspathServer),
333
(LocalTransport, LocalURLServer),
added
removed
bzrlib/transport/local.py
