73
# Cross platform wall-clock time functionality with decent resolution.
74
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
75
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
76
# synchronized with ``time.time()``, this is only meant to be used to find
77
# delta times by subtracting from another call to this function.
78
timer_func = time.time
79
if sys.platform == 'win32':
80
timer_func = time.clock
73
82
# On win32, O_BINARY is used to indicate the file should
74
83
# be opened in binary mode, rather than text mode.
75
84
# On other platforms, O_BINARY doesn't exist, because
98
107
return [a.decode(user_encoding) for a in sys.argv[1:]]
99
108
except UnicodeDecodeError:
100
109
raise errors.BzrError(gettext("Parameter {0!r} encoding is unsupported by {1} "
101
"application locale.").format(a, user_encoding))
110
"application locale.").format(a, user_encoding))
104
113
def make_readonly(filename):
220
225
except OSError as e:
221
226
if e.errno == errno.ENOENT:
224
raise errors.BzrError(
225
gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
229
raise errors.BzrError(gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
228
232
def fancy_rename(old, new, rename_func, unlink_func):
252
256
file_existed = False
254
258
rename_func(new, tmp_name)
255
except (errors.NoSuchFile,):
259
except (errors.NoSuchFile,) as e:
257
261
except IOError as e:
258
262
# RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
263
267
except Exception as e:
264
268
if (getattr(e, 'errno', None) is None
265
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
269
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
268
272
file_existed = True
278
282
# case-insensitive filesystem), so we may have accidentally renamed
279
283
# source by when we tried to rename target
280
284
if (file_existed and e.errno in (None, errno.ENOENT)
281
and old.lower() == new.lower()):
285
and old.lower() == new.lower()):
282
286
# source and target are the same file on a case-insensitive
283
287
# filesystem, so we don't generate an exception
319
323
# as a special case here by simply removing the first slash, as we consider
320
324
# that breaking POSIX compatibility for this obscure feature is acceptable.
321
325
# This is not a paranoid precaution, as we notably get paths like this when
322
# the repo is hosted at the root of the filesystem, i.e. in "/".
326
# the repo is hosted at the root of the filesystem, i.e. in "/".
323
327
if path.startswith('//'):
362
366
return name.decode(user_encoding)
363
367
except UnicodeDecodeError:
364
368
raise errors.BzrError("Encoding of username %r is unsupported by %s "
365
"application locale." % (name, user_encoding))
369
"application locale." % (name, user_encoding))
368
372
def _win32_fixdrive(path):
381
385
def _win32_abspath(path):
382
386
# Real ntpath.abspath doesn't have a problem with a unicode cwd
383
return _win32_fixdrive(ntpath.abspath(path).replace('\\', '/'))
387
return _win32_fixdrive(ntpath.abspath(unicode(path)).replace('\\', '/'))
386
390
def _win32_realpath(path):
387
391
# Real ntpath.realpath doesn't have a problem with a unicode cwd
388
return _win32_fixdrive(ntpath.realpath(path).replace('\\', '/'))
392
return _win32_fixdrive(ntpath.realpath(unicode(path)).replace('\\', '/'))
391
395
def _win32_pathjoin(*args):
438
442
rename_func(old, new)
439
443
except OSError as e:
440
444
detailed_error = OSError(e.errno, e.strerror +
441
" [occurred when renaming '%s' to '%s']" %
445
" [occurred when renaming '%s' to '%s']" %
443
447
detailed_error.old_filename = old
444
448
detailed_error.new_filename = new
445
449
raise detailed_error
510
513
exception = excinfo[1]
511
514
if function in (os.remove, os.rmdir) \
512
and isinstance(exception, OSError) \
513
and exception.errno == errno.EACCES:
515
and isinstance(exception, OSError) \
516
and exception.errno == errno.EACCES:
514
517
make_writable(path)
520
523
"""Replacer for shutil.rmtree: could remove readonly dirs/files"""
521
524
return shutil.rmtree(path, ignore_errors, onerror)
523
get_unicode_argv = getattr(win32utils, 'get_unicode_argv', get_unicode_argv)
526
f = win32utils.get_unicode_argv # special function or None
524
529
path_from_environ = win32utils.get_environ_unicode
525
530
_get_home_dir = win32utils.get_home_location
526
531
getuser_unicode = win32utils.get_user_name
552
557
output_encoding = get_user_encoding()
554
559
mutter('encoding stdout as osutils.get_user_encoding() %r',
557
562
output_encoding = input_encoding
559
564
mutter('encoding stdout as sys.stdin encoding %r',
563
568
mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
566
571
output_encoding = get_user_encoding()
568
573
mutter('cp0 is invalid encoding.'
569
' encoding stdout as osutils.get_user_encoding() %r',
574
' encoding stdout as osutils.get_user_encoding() %r',
573
578
codecs.lookup(output_encoding)
720
719
# writes fail on some platforms (e.g. Windows with SMB mounted
722
721
if not segment_size:
723
segment_size = 5242880 # 5MB
722
segment_size = 5242880 # 5MB
724
723
offsets = range(0, len(bytes), segment_size)
725
724
view = memoryview(bytes)
726
725
write = file_handle.write
727
726
for offset in offsets:
728
write(view[offset:offset + segment_size])
727
write(view[offset:offset+segment_size])
731
730
def file_iterator(input_file, readsize=32768):
831
830
offset = datetime.fromtimestamp(t) - datetime.utcfromtimestamp(t)
832
831
return offset.days * 86400 + offset.seconds
835
833
weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
836
834
_default_format_by_weekday_num = [wd + " %Y-%m-%d %H:%M:%S" for wd in weekdays]
849
847
:param show_offset: Whether to append the timezone.
851
849
(date_fmt, tt, offset_str) = \
852
_format_date(t, offset, timezone, date_fmt, show_offset)
850
_format_date(t, offset, timezone, date_fmt, show_offset)
853
851
date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
854
852
date_str = time.strftime(date_fmt, tt)
855
853
return date_str + offset_str
893
891
:param show_offset: Whether to append the timezone.
895
893
(date_fmt, tt, offset_str) = \
896
_format_date(t, offset, timezone, date_fmt, show_offset)
894
_format_date(t, offset, timezone, date_fmt, show_offset)
897
895
date_str = time.strftime(date_fmt, tt)
898
896
if not isinstance(date_str, text_type):
899
897
date_str = date_str.decode(get_user_encoding(), 'replace')
954
952
plural_seconds = ''
956
954
plural_seconds = 's'
957
if minutes < 90: # print minutes, seconds up to 90 minutes
955
if minutes < 90: # print minutes, seconds up to 90 minutes
959
957
return '%d minute, %d second%s %s' % (
960
minutes, seconds, plural_seconds, direction)
958
minutes, seconds, plural_seconds, direction)
962
960
return '%d minutes, %d second%s %s' % (
963
minutes, seconds, plural_seconds, direction)
961
minutes, seconds, plural_seconds, direction)
965
963
hours = int(minutes / 60)
966
964
minutes -= 60 * hours
975
973
return '%d hours, %d minute%s %s' % (hours, minutes,
976
974
plural_minutes, direction)
980
977
"""Return size of given open file."""
981
978
return os.fstat(f.fileno())[stat.ST_SIZE]
984
# Alias os.urandom to support platforms (which?) without /dev/urandom and
981
# Alias os.urandom to support platforms (which?) without /dev/urandom and
985
982
# override if it doesn't work. Avoid checking on windows where there is
986
983
# significant initialisation cost that can be avoided for some bzr calls.
1022
# TODO: We could later have path objects that remember their list
1023
# decomposition (might be too tricksy though.)
1017
## TODO: We could later have path objects that remember their list
1018
## decomposition (might be too tricksy though.)
1025
1020
def splitpath(p):
1026
1021
"""Turn string into list of parts."""
1027
use_bytes = isinstance(p, bytes)
1028
if os.path.sep == '\\':
1029
# split on either delimiter because people might use either on
1032
ps = re.split(b'[\\\\/]', p)
1034
ps = re.split(r'[\\/]', p)
1043
current_empty_dir = (b'.', b'')
1046
current_empty_dir = ('.', '')
1022
# split on either delimiter because people might use either on
1024
ps = re.split(r'[\\/]', p)
1051
1029
raise errors.BzrError(gettext("sorry, %r not allowed in path") % f)
1052
elif f in current_empty_dir:
1030
elif (f == '.') or (f == ''):
1132
1110
"""Split s into lines, but without removing the newline characters."""
1133
1111
# Trivially convert a fulltext into a 'chunked' representation, and let
1134
1112
# chunks_to_lines do the heavy lifting.
1135
if isinstance(s, bytes):
1113
if isinstance(s, str):
1136
1114
# chunks_to_lines only supports 8-bit strings
1137
1115
return chunks_to_lines([s])
1175
1153
Will delete even if readonly.
1178
_delete_file_or_dir(path)
1156
_delete_file_or_dir(path)
1179
1157
except (OSError, IOError) as e:
1180
1158
if e.errno in (errno.EPERM, errno.EACCES):
1181
1159
# make writable and try again
1194
1172
# - root can damage a solaris file system by using unlink,
1195
1173
# - unlink raises different exceptions on different OSes (linux: EISDIR, win32:
1196
1174
# EACCES, OSX: EPERM) when invoked on a directory.
1197
if isdir(path): # Takes care of symlinks
1175
if isdir(path): # Takes care of symlinks
1200
1178
os.unlink(path)
1330
1308
lbit = bit.lower()
1332
1310
next_entries = _listdir(current)
1333
except OSError: # enoent, eperm, etc
1311
except OSError: # enoent, eperm, etc
1334
1312
# We can't find this in the filesystem, so just append the
1335
1313
# remaining bits.
1336
1314
current = pathjoin(current, bit, *list(bit_iter))
1348
1326
return current[len(abs_base):].lstrip('/')
1351
1328
# XXX - TODO - we need better detection/integration of case-insensitive
1352
1329
# file-systems; Linux often sees FAT32 devices (or NFS-mounted OSX
1353
1330
# filesystems), for example, so could probably benefit from the same basic
1426
1402
:return: None or a utf8 revision id.
1428
1404
if (unicode_or_utf8_string is None
1429
or unicode_or_utf8_string.__class__ == bytes):
1405
or unicode_or_utf8_string.__class__ == bytes):
1430
1406
return unicode_or_utf8_string
1431
1407
raise TypeError('Unicode revision ids are no longer supported. '
1432
1408
'Revision id generators should be creating utf8 revision '
1444
1420
:return: None or a utf8 file id.
1446
1422
if (unicode_or_utf8_string is None
1447
or unicode_or_utf8_string.__class__ == bytes):
1423
or unicode_or_utf8_string.__class__ == bytes):
1448
1424
return unicode_or_utf8_string
1449
1425
raise TypeError('Unicode file ids are no longer supported. '
1450
1426
'File id generators should be creating utf8 file ids.')
1478
1454
can be accessed by that path.
1481
if isinstance(path, bytes):
1482
path = path.decode(sys.getfilesystemencoding())
1483
return unicodedata.normalize('NFC', path), True
1457
return unicodedata.normalize('NFC', text_type(path)), True
1486
1460
def _inaccessible_normalized_filename(path):
1487
1461
__doc__ = _accessible_normalized_filename.__doc__
1489
if isinstance(path, bytes):
1490
path = path.decode(sys.getfilesystemencoding())
1491
normalized = unicodedata.normalize('NFC', path)
1463
normalized = unicodedata.normalize('NFC', text_type(path))
1492
1464
return normalized, normalized == path
1517
1489
except AttributeError:
1518
1490
# siginterrupt doesn't exist on this platform, or for this version
1520
def siginterrupt(signum, flag): return None
1492
siginterrupt = lambda signum, flag: None
1521
1493
if restart_syscall:
1522
1494
def sig_handler(*args):
1523
1495
# Python resets the siginterrupt flag when a signal is
1637
1608
def _win32_terminal_size(width, height):
1638
width, height = win32utils.get_console_size(
1639
defaultx=width, defaulty=height)
1609
width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1640
1610
return width, height
1643
1613
def _ioctl_terminal_size(width, height):
1615
import struct, fcntl, termios
1648
1616
s = struct.pack('HHHH', 0, 0, 0, 0)
1649
1617
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
1650
1618
height, width = struct.unpack('HHHH', x)[0:2]
1668
1635
_terminal_size = _ioctl_terminal_size
1671
def supports_executable(path):
1672
"""Return if filesystem at path supports executable bit.
1674
:param path: Path for which to check the file system
1675
:return: boolean indicating whether executable bit can be stored/relied upon
1677
if sys.platform == 'win32':
1680
fs_type = get_fs_type(path)
1681
except errors.DependencyNotPresent as e:
1682
trace.mutter('Unable to get fs type for %r: %s', path, e)
1684
if fs_type in ('vfat', 'ntfs'):
1685
# filesystems known to not support executable bit
1690
def supports_symlinks(path):
1691
"""Return if the filesystem at path supports the creation of symbolic links.
1694
if not has_symlinks():
1697
fs_type = get_fs_type(path)
1698
except errors.DependencyNotPresent as e:
1699
trace.mutter('Unable to get fs type for %r: %s', path, e)
1701
if fs_type in ('vfat', 'ntfs'):
1702
# filesystems known to not support symlinks
1638
def supports_executable():
1639
return sys.platform != "win32"
1707
1642
def supports_posix_readonly():
1750
1685
raise errors.IllegalPath(path)
1753
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1688
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1756
1690
def _is_error_enotdir(e):
1757
1691
"""Check if this exception represents ENOTDIR.
1769
1703
:return: True if this represents an ENOTDIR error. False otherwise.
1771
1705
en = getattr(e, 'errno', None)
1772
if (en == errno.ENOTDIR or
1773
(sys.platform == 'win32' and
1774
(en == _WIN32_ERROR_DIRECTORY or
1776
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1706
if (en == errno.ENOTDIR
1707
or (sys.platform == 'win32'
1708
and (en == _WIN32_ERROR_DIRECTORY
1709
or (en == errno.EINVAL
1710
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1807
1741
rooted higher up.
1808
1742
:return: an iterator over the dirs.
1810
# TODO there is a bit of a smell where the results of the directory-
1744
#TODO there is a bit of a smell where the results of the directory-
1811
1745
# summary in this, and the path from the root, may not agree
1812
1746
# depending on top and prefix - i.e. ./foo and foo as a pair leads to
1813
1747
# potentially confusing output. We should make this more robust - but
1950
1884
See DirReader.read_dir for details.
1952
1886
_utf8_encode = self._utf8_encode
1954
def _fs_decode(s): return s.decode(_fs_enc)
1956
def _fs_encode(s): return s.encode(_fs_enc)
1957
1887
_lstat = os.lstat
1958
1888
_listdir = os.listdir
1959
1889
_kind_from_mode = file_kind_from_stat_mode
1968
1898
append = dirblock.append
1969
for name_native in _listdir(top.encode('utf-8')):
1899
for name in sorted(_listdir(top)):
1971
name = _fs_decode(name_native)
1901
name_utf8 = _utf8_encode(name)[0]
1972
1902
except UnicodeDecodeError:
1973
1903
raise errors.BadFilenameEncoding(
1974
relprefix + name_native, _fs_enc)
1975
name_utf8 = _utf8_encode(name)[0]
1904
_utf8_encode(relprefix)[0] + name, _fs_enc)
1976
1905
abspath = top_slash + name
1977
1906
statvalue = _lstat(abspath)
1978
1907
kind = _kind_from_mode(statvalue.st_mode)
1979
1908
append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
1980
return sorted(dirblock)
1983
1912
def copy_tree(from_path, to_path, handlers={}):
2041
1970
s = os.stat(src)
2042
1971
chown(dst, s.st_uid, s.st_gid)
1972
except OSError as e:
2045
1974
'Unable to copy ownership from "%s" to "%s". '
2046
1975
'You may want to set it manually.', src, dst)
2059
1988
"""Compare path_a and path_b to generate the same order walkdirs uses."""
2060
1989
key_a = path_prefix_key(path_a)
2061
1990
key_b = path_prefix_key(path_b)
2062
return (key_a > key_b) - (key_a < key_b)
1991
return cmp(key_a, key_b)
2065
1994
_cached_user_encoding = None
2148
2077
def read_bytes_from_socket(sock, report_activity=None,
2149
max_read_size=MAX_SOCKET_CHUNK):
2078
max_read_size=MAX_SOCKET_CHUNK):
2150
2079
"""Read up to max_read_size of bytes from sock and notify of progress.
2152
2081
Translates "Connection reset by peer" into file-like EOF (return an
2158
data = sock.recv(max_read_size)
2087
bytes = sock.recv(max_read_size)
2159
2088
except socket.error as e:
2160
2089
eno = e.args[0]
2161
2090
if eno in _end_of_stream_errors:
2162
2091
# The connection was closed by the other side. Callers expect
2163
2092
# an empty string to signal end-of-stream.
2165
2094
elif eno == errno.EINTR:
2166
2095
# Retry the interrupted recv.
2170
2099
if report_activity is not None:
2171
report_activity(len(data), 'read')
2100
report_activity(len(bytes), 'read')
2175
2104
def recv_all(socket, count):
2209
2138
view = memoryview(bytes)
2210
2139
while sent_total < byte_count:
2212
sent = sock.send(view[sent_total:sent_total + MAX_SOCKET_CHUNK])
2141
sent = sock.send(view[sent_total:sent_total+MAX_SOCKET_CHUNK])
2213
2142
except (socket.error, IOError) as e:
2214
2143
if e.args[0] in _end_of_stream_errors:
2215
2144
raise errors.ConnectionReset(
2293
2221
base = dirname(breezy.__file__)
2294
2222
if getattr(sys, 'frozen', None): # bzr.exe
2295
2223
base = abspath(pathjoin(base, '..', '..'))
2296
with open(pathjoin(base, resource_relpath), "rt") as f:
2224
f = file(pathjoin(base, resource_relpath), "rU")
2297
2226
return f.read()
2300
2230
def file_kind_from_stat_mode_thunk(mode):
2301
2231
global file_kind_from_stat_mode
2304
2234
from ._readdir_pyx import UTF8DirReader
2305
2235
file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
2236
except ImportError as e:
2307
2237
# This is one time where we won't warn that an extension failed to
2308
2238
# load. The extension is never available on Windows anyway.
2309
2239
from ._readdir_py import (
2310
2240
_kind_from_mode as file_kind_from_stat_mode
2312
2242
return file_kind_from_stat_mode(mode)
2315
2243
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2318
2245
def file_stat(f, _lstat=os.lstat):
2324
2251
raise errors.NoSuchFile(f)
2328
2254
def file_kind(f, _lstat=os.lstat):
2329
2255
stat_value = file_stat(f, _lstat)
2330
2256
return file_kind_from_stat_mode(stat_value.st_mode)
2333
2258
def until_no_eintr(f, *a, **kw):
2334
2259
"""Run f(*a, **kw), retrying if an EINTR error occurs.
2386
2311
stdout=subprocess.PIPE).communicate()[0]
2387
2312
elif sys.platform == 'sunos5':
2388
2313
def _local_concurrency():
2389
return subprocess.Popen(['psrinfo', '-p', ],
2314
return subprocess.Popen(['psrinfo', '-p',],
2390
2315
stdout=subprocess.PIPE).communicate()[0]
2391
2316
elif sys.platform == "win32":
2392
2317
def _local_concurrency():
2446
2370
data, _ = self.encode(object, self.errors)
2447
2371
self.stream.write(data)
2450
2373
if sys.platform == 'win32':
2451
2374
def open_file(filename, mode='r', bufsize=-1):
2452
2375
"""This function is used to override the ``open`` builtin.
2527
2450
def find_executable_on_path(name):
2528
2451
"""Finds an executable on the PATH.
2530
2453
On Windows, this will try to append each extension in the PATHEXT
2531
2454
environment variable to the name, if it cannot be found with the name
2534
2457
:param name: The base name of the executable.
2535
2458
:return: The path to the executable found or None.
2574
2497
# exists, though not ours
2577
trace.mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2500
mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2578
2501
# Don't really know.
2581
2504
# Exists and our process: not dead.
2585
2507
if sys.platform == "win32":
2586
2508
is_local_pid_dead = win32utils.is_local_pid_dead
2615
2537
def ensure_empty_directory_exists(path, exception_class):
2616
2538
"""Make sure a local directory exists and is empty.
2618
2540
If it does not exist, it is created. If it exists and is not empty, an
2619
2541
instance of exception_class is raised.
2638
2560
if sys.platform == "win32" and win32utils._is_pywintypes_error(evalue):
2643
def read_mtab(path):
2644
"""Read an fstab-style file and extract mountpoint+filesystem information.
2646
:param path: Path to read from
2647
:yield: Tuples with mountpoints (as bytestrings) and filesystem names
2649
with open(path, 'rb') as f:
2651
if line.startswith(b'#'):
2656
yield cols[1], cols[2].decode('ascii', 'replace')
2659
MTAB_PATH = '/etc/mtab'
2661
class FilesystemFinder(object):
2662
"""Find the filesystem for a particular path."""
2664
def __init__(self, mountpoints):
2667
self._mountpoints = sorted(mountpoints, key=key, reverse=True)
2671
"""Create a FilesystemFinder from an mtab-style file.
2673
Note that this will silenty ignore mtab if it doesn't exist or can not
2676
# TODO(jelmer): Use inotify to be notified when /etc/mtab changes and
2677
# we need to re-read it.
2679
return cls(read_mtab(MTAB_PATH))
2680
except EnvironmentError as e:
2681
trace.mutter('Unable to read mtab: %s', e)
2684
def find(self, path):
2685
"""Find the filesystem used by a particular path.
2687
:param path: Path to find (bytestring or text type)
2688
:return: Filesystem name (as text type) or None, if the filesystem is
2691
for mountpoint, filesystem in self._mountpoints:
2692
if is_inside(mountpoint, path):
2697
_FILESYSTEM_FINDER = None
2700
def get_fs_type(path):
2701
"""Return the filesystem type for the partition a path is in.
2703
:param path: Path to search filesystem type for
2704
:return: A FS type, as string. E.g. "ext2"
2706
global _FILESYSTEM_FINDER
2707
if _FILESYSTEM_FINDER is None:
2708
_FILESYSTEM_FINDER = FilesystemFinder.from_mtab()
2710
if not isinstance(path, bytes):
2711
path = path.encode(_fs_enc)
2713
return _FILESYSTEM_FINDER.find(path)
2717
perf_counter = time.perf_counter
2719
perf_counter = time.clock