44
42
from tempfile import mkdtemp
54
from bzrlib.i18n import gettext
50
from breezy.i18n import gettext
57
from bzrlib.symbol_versioning import (
62
warn as warn_deprecated,
65
53
from hashlib import (
72
from bzrlib import symbol_versioning, _fs_enc
75
# Cross platform wall-clock time functionality with decent resolution.
76
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
77
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
78
# synchronized with ``time.time()``, this is only meant to be used to find
79
# delta times by subtracting from another call to this function.
80
timer_func = time.time
81
if sys.platform == 'win32':
82
timer_func = time.clock
84
66
# On win32, O_BINARY is used to indicate the file should
85
67
# be opened in binary mode, rather than text mode.
92
74
O_NOINHERIT = getattr(os, 'O_NOINHERIT', 0)
95
def get_unicode_argv():
97
user_encoding = get_user_encoding()
98
return [a.decode(user_encoding) for a in sys.argv[1:]]
99
except UnicodeDecodeError:
100
raise errors.BzrError(gettext("Parameter {0!r} encoding is unsupported by {1} "
101
"application locale.").format(a, user_encoding))
77
class UnsupportedTimezoneFormat(errors.BzrError):
79
_fmt = ('Unsupported timezone format "%(timezone)s", '
80
'options are "utc", "original", "local".')
82
def __init__(self, timezone):
83
self.timezone = timezone
104
86
def make_readonly(filename):
105
87
"""Make a filename read-only."""
106
88
mod = os.lstat(filename).st_mode
107
89
if not stat.S_ISLNK(mod):
109
91
chmod_if_possible(filename, mod)
112
94
def make_writable(filename):
113
95
mod = os.lstat(filename).st_mode
114
96
if not stat.S_ISLNK(mod):
116
98
chmod_if_possible(filename, mod)
124
106
# It is probably faster to just do the chmod, rather than
125
107
# doing a stat, and then trying to compare
126
108
os.chmod(filename, mode)
127
except (IOError, OSError),e:
109
except (IOError, OSError) as e:
128
110
# Permission/access denied seems to commonly happen on smbfs; there's
129
111
# probably no point warning about it.
130
112
# <https://bugs.launchpad.net/bzr/+bug/606537>
213
199
stat = getattr(os, 'lstat', os.stat)
217
203
if e.errno == errno.ENOENT:
220
raise errors.BzrError(gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
206
raise errors.BzrError(
207
gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
223
210
def fancy_rename(old, new, rename_func, unlink_func):
247
234
file_existed = False
249
236
rename_func(new, tmp_name)
250
except (errors.NoSuchFile,), e:
237
except (errors.NoSuchFile,):
253
240
# RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
254
241
# function raises an IOError with errno is None when a rename fails.
255
242
# This then gets caught here.
256
243
if e.errno not in (None, errno.ENOENT, errno.ENOTDIR):
245
except Exception as e:
259
246
if (getattr(e, 'errno', None) is None
260
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
247
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
263
250
file_existed = True
269
# This may throw an exception, in which case success will
271
rename_func(old, new)
273
except (IOError, OSError), e:
274
# source and target may be aliases of each other (e.g. on a
275
# case-insensitive filesystem), so we may have accidentally renamed
276
# source by when we tried to rename target
277
failure_exc = sys.exc_info()
278
if (file_existed and e.errno in (None, errno.ENOENT)
254
# This may throw an exception, in which case success will
256
rename_func(old, new)
258
except (IOError, OSError) as e:
259
# source and target may be aliases of each other (e.g. on a
260
# case-insensitive filesystem), so we may have accidentally renamed
261
# source by when we tried to rename target
262
if (file_existed and e.errno in (None, errno.ENOENT)
279
263
and old.lower() == new.lower()):
280
# source and target are the same file on a case-insensitive
281
# filesystem, so we don't generate an exception
264
# source and target are the same file on a case-insensitive
265
# filesystem, so we don't generate an exception
285
271
# If the file used to exist, rename it back into place
320
301
# as a special case here by simply removing the first slash, as we consider
321
302
# that breaking POSIX compatibility for this obscure feature is acceptable.
322
303
# This is not a paranoid precaution, as we notably get paths like this when
323
# the repo is hosted at the root of the filesystem, i.e. in "/".
304
# the repo is hosted at the root of the filesystem, i.e. in "/".
324
305
if path.startswith('//'):
329
def _posix_path_from_environ(key):
330
"""Get unicode path from `key` in environment or None if not present
332
Note that posix systems use arbitrary byte strings for filesystem objects,
333
so a path that raises BadFilenameEncoding here may still be accessible.
335
val = os.environ.get(key, None)
339
return val.decode(_fs_enc)
340
except UnicodeDecodeError:
341
# GZ 2011-12-12:Ideally want to include `key` in the exception message
342
raise errors.BadFilenameEncoding(val, _fs_enc)
345
310
def _posix_get_home_dir():
346
311
"""Get the home directory of the current user as a unicode path"""
347
312
path = posixpath.expanduser("~")
349
314
return path.decode(_fs_enc)
315
except AttributeError:
350
317
except UnicodeDecodeError:
351
318
raise errors.BadFilenameEncoding(path, _fs_enc)
354
321
def _posix_getuser_unicode():
355
322
"""Get username from environment or password database as unicode"""
356
name = getpass.getuser()
357
user_encoding = get_user_encoding()
359
return name.decode(user_encoding)
360
except UnicodeDecodeError:
361
raise errors.BzrError("Encoding of username %r is unsupported by %s "
362
"application locale." % (name, user_encoding))
323
return getpass.getuser()
365
326
def _win32_fixdrive(path):
378
339
def _win32_abspath(path):
379
340
# Real ntpath.abspath doesn't have a problem with a unicode cwd
380
return _win32_fixdrive(ntpath.abspath(unicode(path)).replace('\\', '/'))
383
def _win98_abspath(path):
384
"""Return the absolute version of a path.
385
Windows 98 safe implementation (python reimplementation
386
of Win32 API function GetFullPathNameW)
391
# \\HOST\path => //HOST/path
392
# //HOST/path => //HOST/path
393
# path => C:/cwd/path
396
# check for absolute path
397
drive = ntpath.splitdrive(path)[0]
398
if drive == '' and path[:2] not in('//','\\\\'):
400
# we cannot simply os.path.join cwd and path
401
# because os.path.join('C:','/path') produce '/path'
402
# and this is incorrect
403
if path[:1] in ('/','\\'):
404
cwd = ntpath.splitdrive(cwd)[0]
406
path = cwd + '\\' + path
407
return _win32_fixdrive(ntpath.normpath(path).replace('\\', '/'))
341
return _win32_fixdrive(ntpath.abspath(path).replace('\\', '/'))
410
344
def _win32_realpath(path):
411
345
# Real ntpath.realpath doesn't have a problem with a unicode cwd
412
return _win32_fixdrive(ntpath.realpath(unicode(path)).replace('\\', '/'))
346
return _win32_fixdrive(ntpath.realpath(path).replace('\\', '/'))
415
349
def _win32_pathjoin(*args):
419
353
def _win32_normpath(path):
420
return _win32_fixdrive(ntpath.normpath(unicode(path)).replace('\\', '/'))
354
return _win32_fixdrive(ntpath.normpath(path).replace('\\', '/'))
423
357
def _win32_getcwd():
424
return _win32_fixdrive(os.getcwdu().replace('\\', '/'))
358
return _win32_fixdrive(_getcwd().replace('\\', '/'))
427
361
def _win32_mkdtemp(*args, **kwargs):
438
372
fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
440
374
if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY, errno.EINVAL):
441
375
# If we try to rename a non-existant file onto cwd, we get
442
376
# EPERM or EACCES instead of ENOENT, this will raise ENOENT
460
394
def _rename_wrapper(old, new):
462
396
rename_func(old, new)
464
398
detailed_error = OSError(e.errno, e.strerror +
465
" [occurred when renaming '%s' to '%s']" %
399
" [occurred when renaming '%s' to '%s']" %
467
401
detailed_error.old_filename = old
468
402
detailed_error.new_filename = new
469
403
raise detailed_error
471
405
return _rename_wrapper
473
411
# Default rename wraps os.rename()
474
412
rename = _rename_wrap_exception(os.rename)
587
518
output_encoding = get_user_encoding()
589
520
mutter('cp0 is invalid encoding.'
590
' encoding stdout as osutils.get_user_encoding() %r',
521
' encoding stdout as osutils.get_user_encoding() %r',
594
525
codecs.lookup(output_encoding)
595
526
except LookupError:
596
sys.stderr.write('bzr: warning:'
527
sys.stderr.write('brz: warning:'
597
528
' unknown terminal encoding %s.\n'
598
529
' Using encoding %s instead.\n'
599
530
% (output_encoding, get_user_encoding())
601
532
output_encoding = get_user_encoding()
603
534
return output_encoding
735
672
# writes fail on some platforms (e.g. Windows with SMB mounted
737
674
if not segment_size:
738
segment_size = 5242880 # 5MB
739
segments = range(len(bytes) / segment_size + 1)
675
segment_size = 5242880 # 5MB
676
offsets = range(0, len(bytes), segment_size)
677
view = memoryview(bytes)
740
678
write = file_handle.write
741
for segment_index in segments:
742
segment = buffer(bytes, segment_index * segment_size, segment_size)
679
for offset in offsets:
680
write(view[offset:offset + segment_size])
746
683
def file_iterator(input_file, readsize=32768):
801
744
def sha_strings(strings, _factory=sha):
802
745
"""Return the sha-1 of concatenation of strings"""
804
map(s.update, strings)
747
for string in strings:
808
752
def sha_string(f, _factory=sha):
809
return _factory(f).hexdigest()
753
# GZ 2017-09-16: Dodgy if factory is ever not sha, probably shouldn't be.
754
return _hexdigest(_factory(f))
812
757
def fingerprint_file(f):
814
759
return {'size': len(b),
815
'sha1': sha(b).hexdigest()}
760
'sha1': _hexdigest(sha(b))}
818
763
def compare_files(a, b):
851
797
:param show_offset: Whether to append the timezone.
853
799
(date_fmt, tt, offset_str) = \
854
_format_date(t, offset, timezone, date_fmt, show_offset)
800
_format_date(t, offset, timezone, date_fmt, show_offset)
855
801
date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
856
802
date_str = time.strftime(date_fmt, tt)
857
803
return date_str + offset_str
895
841
:param show_offset: Whether to append the timezone.
897
843
(date_fmt, tt, offset_str) = \
898
_format_date(t, offset, timezone, date_fmt, show_offset)
844
_format_date(t, offset, timezone, date_fmt, show_offset)
899
845
date_str = time.strftime(date_fmt, tt)
900
if not isinstance(date_str, unicode):
846
if not isinstance(date_str, str):
901
847
date_str = date_str.decode(get_user_encoding(), 'replace')
902
848
return date_str + offset_str
956
902
plural_seconds = ''
958
904
plural_seconds = 's'
959
if minutes < 90: # print minutes, seconds up to 90 minutes
905
if minutes < 90: # print minutes, seconds up to 90 minutes
961
907
return '%d minute, %d second%s %s' % (
962
minutes, seconds, plural_seconds, direction)
908
minutes, seconds, plural_seconds, direction)
964
910
return '%d minutes, %d second%s %s' % (
965
minutes, seconds, plural_seconds, direction)
911
minutes, seconds, plural_seconds, direction)
967
913
hours = int(minutes / 60)
968
914
minutes -= 60 * hours
977
923
return '%d hours, %d minute%s %s' % (hours, minutes,
978
924
plural_minutes, direction)
981
928
"""Return size of given open file."""
982
929
return os.fstat(f.fileno())[stat.ST_SIZE]
985
# Alias os.urandom to support platforms (which?) without /dev/urandom and
932
# Alias os.urandom to support platforms (which?) without /dev/urandom and
986
933
# override if it doesn't work. Avoid checking on windows where there is
987
934
# significant initialisation cost that can be avoided for some bzr calls.
1013
962
for raw_byte in rand_bytes(num):
1014
s += ALNUM[ord(raw_byte) % 36]
963
s += ALNUM[raw_byte % 36]
1018
## TODO: We could later have path objects that remember their list
1019
## decomposition (might be too tricksy though.)
967
# TODO: We could later have path objects that remember their list
968
# decomposition (might be too tricksy though.)
1021
970
def splitpath(p):
1022
971
"""Turn string into list of parts."""
1023
# split on either delimiter because people might use either on
1025
ps = re.split(r'[\\/]', p)
972
use_bytes = isinstance(p, bytes)
973
if os.path.sep == '\\':
974
# split on either delimiter because people might use either on
977
ps = re.split(b'[\\\\/]', p)
979
ps = re.split(r'[\\/]', p)
988
current_empty_dir = (b'.', b'')
991
current_empty_dir = ('.', '')
1030
996
raise errors.BzrError(gettext("sorry, %r not allowed in path") % f)
1031
elif (f == '.') or (f == ''):
997
elif f in current_empty_dir:
1066
1032
implementation should be loaded instead::
1069
>>> import bzrlib._fictional_extension_pyx
1035
>>> import breezy._fictional_extension_pyx
1070
1036
>>> except ImportError, e:
1071
>>> bzrlib.osutils.failed_to_load_extension(e)
1072
>>> import bzrlib._fictional_extension_py
1037
>>> breezy.osutils.failed_to_load_extension(e)
1038
>>> import breezy._fictional_extension_py
1074
1040
# NB: This docstring is just an example, not a doctest, because doctest
1075
1041
# currently can't cope with the use of lazy imports in this namespace --
1088
1054
def report_extension_load_failures():
1089
1055
if not _extension_load_failures:
1091
if config.GlobalStack().get('ignore_missing_extensions'):
1057
if config.GlobalConfig().suppress_warning('missing_extensions'):
1093
1059
# the warnings framework should by default show this only once
1094
from bzrlib.trace import warning
1060
from .trace import warning
1096
"bzr: warning: some compiled extensions could not be loaded; "
1097
"see <https://answers.launchpad.net/bzr/+faq/703>")
1062
"brz: warning: some compiled extensions could not be loaded; "
1063
"see ``brz help missing-extensions``")
1098
1064
# we no longer show the specific missing extensions here, because it makes
1099
1065
# the message too long and scary - see
1100
1066
# https://bugs.launchpad.net/bzr/+bug/430529
1104
from bzrlib._chunks_to_lines_pyx import chunks_to_lines
1105
except ImportError, e:
1070
from ._chunks_to_lines_pyx import chunks_to_lines
1071
except ImportError as e:
1106
1072
failed_to_load_extension(e)
1107
from bzrlib._chunks_to_lines_py import chunks_to_lines
1073
from ._chunks_to_lines_py import chunks_to_lines
1110
1076
def split_lines(s):
1111
1077
"""Split s into lines, but without removing the newline characters."""
1112
1078
# Trivially convert a fulltext into a 'chunked' representation, and let
1113
1079
# chunks_to_lines do the heavy lifting.
1114
if isinstance(s, str):
1080
if isinstance(s, bytes):
1115
1081
# chunks_to_lines only supports 8-bit strings
1116
1082
return chunks_to_lines([s])
1153
1120
Will delete even if readonly.
1156
_delete_file_or_dir(path)
1157
except (OSError, IOError), e:
1123
_delete_file_or_dir(path)
1124
except (OSError, IOError) as e:
1158
1125
if e.errno in (errno.EPERM, errno.EACCES):
1159
1126
# make writable and try again
1221
1188
# 3) '\xa0' isn't unicode safe since it is >128.
1223
# This should *not* be a unicode set of characters in case the source
1224
# string is not a Unicode string. We can auto-up-cast the characters since
1225
# they are ascii, but we don't want to auto-up-cast the string in case it
1227
for ch in ' \t\n\r\v\f':
1190
if isinstance(s, str):
1193
ws = (b' ', b'\t', b'\n', b'\r', b'\v', b'\f')
1301
1268
abs_base = abspath(base)
1302
1269
current = abs_base
1303
_listdir = os.listdir
1305
1271
# use an explicit iterator so we can easily consume the rest on early exit.
1306
1272
bit_iter = iter(rel.split('/'))
1307
1273
for bit in bit_iter:
1308
1274
lbit = bit.lower()
1310
next_entries = _listdir(current)
1311
except OSError: # enoent, eperm, etc
1276
next_entries = scandir(current)
1277
except OSError: # enoent, eperm, etc
1312
1278
# We can't find this in the filesystem, so just append the
1313
1279
# remaining bits.
1314
1280
current = pathjoin(current, bit, *list(bit_iter))
1316
for look in next_entries:
1317
if lbit == look.lower():
1318
current = pathjoin(current, look)
1282
for entry in next_entries:
1283
if lbit == entry.name.lower():
1284
current = entry.path
1321
1287
# got to the end, nothing matched, so we just return the
1367
1335
Otherwise it is decoded from utf-8. If decoding fails, the exception is
1368
1336
wrapped in a BzrBadParameterNotUnicode exception.
1370
if isinstance(unicode_or_utf8_string, unicode):
1338
if isinstance(unicode_or_utf8_string, str):
1371
1339
return unicode_or_utf8_string
1373
1341
return unicode_or_utf8_string.decode('utf8')
1381
1349
If it is a str, it is returned.
1382
1350
If it is Unicode, it is encoded into a utf-8 string.
1384
if isinstance(unicode_or_utf8_string, str):
1352
if isinstance(unicode_or_utf8_string, bytes):
1385
1353
# TODO: jam 20070209 This is overkill, and probably has an impact on
1386
1354
# performance if we are dealing with lots of apis that want a
1387
1355
# utf-8 revision id
1394
1362
return unicode_or_utf8_string.encode('utf-8')
1397
_revision_id_warning = ('Unicode revision ids were deprecated in bzr 0.15.'
1398
' Revision id generators should be creating utf8'
1402
def safe_revision_id(unicode_or_utf8_string, warn=True):
1403
"""Revision ids should now be utf8, but at one point they were unicode.
1405
:param unicode_or_utf8_string: A possibly Unicode revision_id. (can also be
1407
:param warn: Functions that are sanitizing user data can set warn=False
1408
:return: None or a utf8 revision id.
1410
if (unicode_or_utf8_string is None
1411
or unicode_or_utf8_string.__class__ == str):
1412
return unicode_or_utf8_string
1414
symbol_versioning.warn(_revision_id_warning, DeprecationWarning,
1416
return cache_utf8.encode(unicode_or_utf8_string)
1419
_file_id_warning = ('Unicode file ids were deprecated in bzr 0.15. File id'
1420
' generators should be creating utf8 file ids.')
1423
def safe_file_id(unicode_or_utf8_string, warn=True):
1424
"""File ids should now be utf8, but at one point they were unicode.
1426
This is the same as safe_utf8, except it uses the cached encode functions
1427
to save a little bit of performance.
1429
:param unicode_or_utf8_string: A possibly Unicode file_id. (can also be
1431
:param warn: Functions that are sanitizing user data can set warn=False
1432
:return: None or a utf8 file id.
1434
if (unicode_or_utf8_string is None
1435
or unicode_or_utf8_string.__class__ == str):
1436
return unicode_or_utf8_string
1438
symbol_versioning.warn(_file_id_warning, DeprecationWarning,
1440
return cache_utf8.encode(unicode_or_utf8_string)
1443
1365
_platform_normalizes_filenames = False
1444
1366
if sys.platform == 'darwin':
1445
1367
_platform_normalizes_filenames = True
1468
1390
can be accessed by that path.
1471
return unicodedata.normalize('NFC', unicode(path)), True
1393
if isinstance(path, bytes):
1394
path = path.decode(sys.getfilesystemencoding())
1395
return unicodedata.normalize('NFC', path), True
1474
1398
def _inaccessible_normalized_filename(path):
1475
1399
__doc__ = _accessible_normalized_filename.__doc__
1477
normalized = unicodedata.normalize('NFC', unicode(path))
1401
if isinstance(path, bytes):
1402
path = path.decode(sys.getfilesystemencoding())
1403
normalized = unicodedata.normalize('NFC', path)
1478
1404
return normalized, normalized == path
1534
1460
_terminal_size_state = 'no_data'
1535
1461
_first_terminal_size = None
1537
1464
def terminal_width():
1538
1465
"""Return terminal width.
1540
1467
None is returned if the width can't established precisely.
1543
- if BZR_COLUMNS is set, returns its value
1470
- if BRZ_COLUMNS is set, returns its value
1544
1471
- if there is no controlling terminal, returns None
1545
1472
- query the OS, if the queried size has changed since the last query,
1546
1473
return its value,
1561
1488
# Note to implementors: if changing the rules for determining the width,
1562
1489
# make sure you've considered the behaviour in these cases:
1563
1490
# - M-x shell in emacs, where $COLUMNS is set and TIOCGWINSZ returns 0,0.
1564
# - bzr log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1491
# - brz log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1566
1493
# - (add more interesting cases here, if you find any)
1567
1494
# Some programs implement "Use $COLUMNS (if set) until SIGWINCH occurs",
1571
1498
# time so we can notice if the reported size has changed, which should have
1572
1499
# a similar effect.
1574
# If BZR_COLUMNS is set, take it, user is always right
1501
# If BRZ_COLUMNS is set, take it, user is always right
1575
1502
# Except if they specified 0 in which case, impose no limit here
1577
width = int(os.environ['BZR_COLUMNS'])
1504
width = int(os.environ['BRZ_COLUMNS'])
1578
1505
except (KeyError, ValueError):
1580
1507
if width is not None:
1622
1549
def _win32_terminal_size(width, height):
1623
width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1550
width, height = win32utils.get_console_size(
1551
defaultx=width, defaulty=height)
1624
1552
return width, height
1627
1555
def _ioctl_terminal_size(width, height):
1629
import struct, fcntl, termios
1630
1560
s = struct.pack('HHHH', 0, 0, 0, 0)
1631
1561
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
1632
1562
height, width = struct.unpack('HHHH', x)[0:2]
1649
1580
_terminal_size = _ioctl_terminal_size
1652
def supports_executable():
1653
return sys.platform != "win32"
1583
def supports_executable(path):
1584
"""Return if filesystem at path supports executable bit.
1586
:param path: Path for which to check the file system
1587
:return: boolean indicating whether executable bit can be stored/relied upon
1589
if sys.platform == 'win32':
1592
fs_type = get_fs_type(path)
1593
except errors.DependencyNotPresent as e:
1594
trace.mutter('Unable to get fs type for %r: %s', path, e)
1596
if fs_type in ('vfat', 'ntfs'):
1597
# filesystems known to not support executable bit
1602
def supports_symlinks(path):
1603
"""Return if the filesystem at path supports the creation of symbolic links.
1606
if not has_symlinks():
1609
fs_type = get_fs_type(path)
1610
except errors.DependencyNotPresent as e:
1611
trace.mutter('Unable to get fs type for %r: %s', path, e)
1613
if fs_type in ('vfat', 'ntfs'):
1614
# filesystems known to not support symlinks
1656
1619
def supports_posix_readonly():
1699
1660
raise errors.IllegalPath(path)
1702
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1663
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1667
scandir = os.scandir
1668
except AttributeError: # Python < 3
1669
lazy_import(globals(), """\
1670
from scandir import scandir
1704
1674
def _is_error_enotdir(e):
1705
1675
"""Check if this exception represents ENOTDIR.
1717
1687
:return: True if this represents an ENOTDIR error. False otherwise.
1719
1689
en = getattr(e, 'errno', None)
1720
if (en == errno.ENOTDIR
1721
or (sys.platform == 'win32'
1722
and (en == _WIN32_ERROR_DIRECTORY
1723
or (en == errno.EINVAL
1724
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1690
if (en == errno.ENOTDIR or
1691
(sys.platform == 'win32' and
1692
(en == _WIN32_ERROR_DIRECTORY or
1694
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1755
1725
rooted higher up.
1756
1726
:return: an iterator over the dirs.
1758
#TODO there is a bit of a smell where the results of the directory-
1728
# TODO there is a bit of a smell where the results of the directory-
1759
1729
# summary in this, and the path from the root, may not agree
1760
1730
# depending on top and prefix - i.e. ./foo and foo as a pair leads to
1761
1731
# potentially confusing output. We should make this more robust - but
1762
1732
# not at a speed cost. RBC 20060731
1764
1733
_directory = _directory_kind
1765
_listdir = os.listdir
1766
_kind_from_mode = file_kind_from_stat_mode
1767
1734
pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
1769
1736
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1775
1742
top_slash = top + u'/'
1778
append = dirblock.append
1780
names = sorted(map(decode_filename, _listdir(top)))
1746
for entry in scandir(top):
1747
name = decode_filename(entry.name)
1748
statvalue = entry.stat(follow_symlinks=False)
1749
kind = file_kind_from_stat_mode(statvalue.st_mode)
1750
dirblock.append((relprefix + name, name, kind, statvalue, entry.path))
1751
except OSError as e:
1782
1752
if not _is_error_enotdir(e):
1786
abspath = top_slash + name
1787
statvalue = _lstat(abspath)
1788
kind = _kind_from_mode(statvalue.st_mode)
1789
append((relprefix + name, name, kind, statvalue, abspath))
1754
except UnicodeDecodeError as e:
1755
raise errors.BadFilenameEncoding(e.object, _fs_enc)
1790
1757
yield (relroot, top), dirblock
1792
1759
# push the user specified dirs from dirblock
1838
1805
global _selected_dir_reader
1839
1806
if _selected_dir_reader is None:
1840
if sys.platform == "win32" and win32utils.winver == 'Windows NT':
1841
# Win98 doesn't have unicode apis like FindFirstFileW
1842
# TODO: We possibly could support Win98 by falling back to the
1843
# original FindFirstFile, and using TCHAR instead of WCHAR,
1844
# but that gets a bit tricky, and requires custom compiling
1807
if sys.platform == "win32":
1847
from bzrlib._walkdirs_win32 import Win32ReadDir
1809
from ._walkdirs_win32 import Win32ReadDir
1848
1810
_selected_dir_reader = Win32ReadDir()
1849
1811
except ImportError:
1851
1813
elif _fs_enc in ('utf-8', 'ascii'):
1853
from bzrlib._readdir_pyx import UTF8DirReader
1815
from ._readdir_pyx import UTF8DirReader
1854
1816
_selected_dir_reader = UTF8DirReader()
1855
except ImportError, e:
1817
except ImportError as e:
1856
1818
failed_to_load_extension(e)
1903
1865
See DirReader.read_dir for details.
1905
1867
_utf8_encode = self._utf8_encode
1907
_listdir = os.listdir
1908
_kind_from_mode = file_kind_from_stat_mode
1869
def _fs_decode(s): return s.decode(_fs_enc)
1871
def _fs_encode(s): return s.encode(_fs_enc)
1911
relprefix = prefix + '/'
1874
relprefix = prefix + b'/'
1914
top_slash = top + u'/'
1877
top_slash = top + '/'
1917
1880
append = dirblock.append
1918
for name in sorted(_listdir(top)):
1881
for entry in scandir(safe_utf8(top)):
1920
name_utf8 = _utf8_encode(name)[0]
1883
name = _fs_decode(entry.name)
1921
1884
except UnicodeDecodeError:
1922
1885
raise errors.BadFilenameEncoding(
1923
_utf8_encode(relprefix)[0] + name, _fs_enc)
1886
relprefix + entry.name, _fs_enc)
1924
1887
abspath = top_slash + name
1925
statvalue = _lstat(abspath)
1926
kind = _kind_from_mode(statvalue.st_mode)
1888
name_utf8 = _utf8_encode(name)[0]
1889
statvalue = entry.stat(follow_symlinks=False)
1890
kind = file_kind_from_stat_mode(statvalue.st_mode)
1927
1891
append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
1892
return sorted(dirblock)
1931
1895
def copy_tree(from_path, to_path, handlers={}):
2001
1965
This can be used to sort paths in the same way that walkdirs does.
2003
return (dirname(path) , path)
1967
return (dirname(path), path)
2006
1970
def compare_paths_prefix_order(path_a, path_b):
2007
1971
"""Compare path_a and path_b to generate the same order walkdirs uses."""
2008
1972
key_a = path_prefix_key(path_a)
2009
1973
key_b = path_prefix_key(path_b)
2010
return cmp(key_a, key_b)
1974
return (key_a > key_b) - (key_a < key_b)
2013
1977
_cached_user_encoding = None
2016
def get_user_encoding(use_cache=DEPRECATED_PARAMETER):
1980
def get_user_encoding():
2017
1981
"""Find out what the preferred user encoding is.
2019
1983
This is generally the encoding that is used for command line parameters
2023
1987
:return: A string defining the preferred user encoding
2025
1989
global _cached_user_encoding
2026
if deprecated_passed(use_cache):
2027
warn_deprecated("use_cache should only have been used for tests",
2028
DeprecationWarning, stacklevel=2)
2029
1990
if _cached_user_encoding is not None:
2030
1991
return _cached_user_encoding
2097
2058
def read_bytes_from_socket(sock, report_activity=None,
2098
max_read_size=MAX_SOCKET_CHUNK):
2059
max_read_size=MAX_SOCKET_CHUNK):
2099
2060
"""Read up to max_read_size of bytes from sock and notify of progress.
2101
2062
Translates "Connection reset by peer" into file-like EOF (return an
2102
2063
empty string rather than raise an error), and repeats the recv if
2103
2064
interrupted by a signal.
2107
bytes = sock.recv(max_read_size)
2108
except socket.error, e:
2068
data = sock.recv(max_read_size)
2069
except socket.error as e:
2109
2070
eno = e.args[0]
2110
2071
if eno in _end_of_stream_errors:
2111
2072
# The connection was closed by the other side. Callers expect
2112
2073
# an empty string to signal end-of-stream.
2114
2075
elif eno == errno.EINTR:
2115
2076
# Retry the interrupted recv.
2119
2080
if report_activity is not None:
2120
report_activity(len(bytes), 'read')
2081
report_activity(len(data), 'read')
2124
2085
def recv_all(socket, count):
2157
2118
byte_count = len(bytes)
2119
view = memoryview(bytes)
2158
2120
while sent_total < byte_count:
2160
sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))
2161
except (socket.error, IOError), e:
2122
sent = sock.send(view[sent_total:sent_total + MAX_SOCKET_CHUNK])
2123
except (socket.error, IOError) as e:
2162
2124
if e.args[0] in _end_of_stream_errors:
2163
2125
raise errors.ConnectionReset(
2164
2126
"Error trying to write to socket", e)
2227
2190
If and when pkg_resources becomes a standard library, this routine
2228
2191
can delegate to it.
2230
# Check package name is within bzrlib
2231
if package == "bzrlib":
2193
# Check package name is within breezy
2194
if package == "breezy":
2232
2195
resource_relpath = resource_name
2233
elif package.startswith("bzrlib."):
2234
package = package[len("bzrlib."):].replace('.', os.sep)
2196
elif package.startswith("breezy."):
2197
package = package[len("breezy."):].replace('.', os.sep)
2235
2198
resource_relpath = pathjoin(package, resource_name)
2237
raise errors.BzrError('resource package %s not in bzrlib' % package)
2200
raise errors.BzrError('resource package %s not in breezy' % package)
2239
2202
# Map the resource to a file and read its contents
2240
base = dirname(bzrlib.__file__)
2203
base = dirname(breezy.__file__)
2241
2204
if getattr(sys, 'frozen', None): # bzr.exe
2242
2205
base = abspath(pathjoin(base, '..', '..'))
2243
f = file(pathjoin(base, resource_relpath), "rU")
2206
with open(pathjoin(base, resource_relpath), "rt") as f:
2245
2207
return f.read()
2249
2210
def file_kind_from_stat_mode_thunk(mode):
2250
2211
global file_kind_from_stat_mode
2251
2212
if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
2253
from bzrlib._readdir_pyx import UTF8DirReader
2214
from ._readdir_pyx import UTF8DirReader
2254
2215
file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
2255
except ImportError, e:
2256
2217
# This is one time where we won't warn that an extension failed to
2257
2218
# load. The extension is never available on Windows anyway.
2258
from bzrlib._readdir_py import (
2219
from ._readdir_py import (
2259
2220
_kind_from_mode as file_kind_from_stat_mode
2261
2222
return file_kind_from_stat_mode(mode)
2262
2225
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2264
2228
def file_stat(f, _lstat=os.lstat):
2267
2231
return _lstat(f)
2232
except OSError as e:
2269
2233
if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
2270
2234
raise errors.NoSuchFile(f)
2273
2238
def file_kind(f, _lstat=os.lstat):
2274
2239
stat_value = file_stat(f, _lstat)
2275
2240
return file_kind_from_stat_mode(stat_value.st_mode)
2277
2243
def until_no_eintr(f, *a, **kw):
2278
2244
"""Run f(*a, **kw), retrying if an EINTR error occurs.
2284
2250
Keep in mind that this is not a complete solution to EINTR. There is
2285
2251
probably code in the Python standard library and other dependencies that
2286
2252
may encounter EINTR if a signal arrives (and there is signal handler for
2287
that signal). So this function can reduce the impact for IO that bzrlib
2253
that signal). So this function can reduce the impact for IO that breezy
2288
2254
directly controls, but it is not a complete solution.
2290
2256
# Borrowed from Twisted's twisted.python.util.untilConcludes function.
2293
2259
return f(*a, **kw)
2294
except (IOError, OSError), e:
2260
except (IOError, OSError) as e:
2295
2261
if e.errno == errno.EINTR:
2300
@deprecated_function(deprecated_in((2, 2, 0)))
2301
def re_compile_checked(re_string, flags=0, where=""):
2302
"""Return a compiled re, or raise a sensible error.
2304
This should only be used when compiling user-supplied REs.
2306
:param re_string: Text form of regular expression.
2307
:param flags: eg re.IGNORECASE
2308
:param where: Message explaining to the user the context where
2309
it occurred, eg 'log search filter'.
2311
# from https://bugs.launchpad.net/bzr/+bug/251352
2313
re_obj = re.compile(re_string, flags)
2316
except errors.InvalidPattern, e:
2318
where = ' in ' + where
2319
# despite the name 'error' is a type
2320
raise errors.BzrCommandError('Invalid regular expression%s: %s'
2324
2266
if sys.platform == "win32":
2354
2296
stdout=subprocess.PIPE).communicate()[0]
2355
2297
elif sys.platform == 'sunos5':
2356
2298
def _local_concurrency():
2357
return subprocess.Popen(['psrinfo', '-p',],
2299
return subprocess.Popen(['psrinfo', '-p', ],
2358
2300
stdout=subprocess.PIPE).communicate()[0]
2359
2301
elif sys.platform == "win32":
2360
2302
def _local_concurrency():
2379
2322
if _cached_local_concurrency is not None and use_cache:
2380
2323
return _cached_local_concurrency
2382
concurrency = os.environ.get('BZR_CONCURRENCY', None)
2325
concurrency = os.environ.get('BRZ_CONCURRENCY', None)
2383
2326
if concurrency is None:
2327
import multiprocessing
2385
import multiprocessing
2386
2329
concurrency = multiprocessing.cpu_count()
2387
except (ImportError, NotImplementedError):
2388
# multiprocessing is only available on Python >= 2.6
2389
# and multiprocessing.cpu_count() isn't implemented on all
2330
except NotImplementedError:
2331
# multiprocessing.cpu_count() isn't implemented on all platforms
2392
2333
concurrency = _local_concurrency()
2393
2334
except (OSError, IOError):
2409
2350
self.encode = encode
2411
2352
def write(self, object):
2412
if type(object) is str:
2353
if isinstance(object, str):
2413
2354
self.stream.write(object)
2415
2356
data, _ = self.encode(object, self.errors)
2416
2357
self.stream.write(data)
2418
2360
if sys.platform == 'win32':
2419
2361
def open_file(filename, mode='r', bufsize=-1):
2420
2362
"""This function is used to override the ``open`` builtin.
2542
2484
# exists, though not ours
2545
mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2487
trace.mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2546
2488
# Don't really know.
2549
2491
# Exists and our process: not dead.
2552
2495
if sys.platform == "win32":
2553
2496
is_local_pid_dead = win32utils.is_local_pid_dead
2582
2525
def ensure_empty_directory_exists(path, exception_class):
2583
2526
"""Make sure a local directory exists and is empty.
2585
2528
If it does not exist, it is created. If it exists and is not empty, an
2586
2529
instance of exception_class is raised.
2533
except OSError as e:
2591
2534
if e.errno != errno.EEXIST:
2593
2536
if os.listdir(path) != []:
2594
2537
raise exception_class(path)
2597
def is_environment_error(evalue):
2598
"""True if exception instance is due to a process environment issue
2600
This includes OSError and IOError, but also other errors that come from
2601
the operating system or core libraries but are not subclasses of those.
2603
if isinstance(evalue, (EnvironmentError, select.error)):
2605
if sys.platform == "win32" and win32utils._is_pywintypes_error(evalue):
2540
def read_mtab(path):
2541
"""Read an fstab-style file and extract mountpoint+filesystem information.
2543
:param path: Path to read from
2544
:yield: Tuples with mountpoints (as bytestrings) and filesystem names
2546
with open(path, 'rb') as f:
2548
if line.startswith(b'#'):
2553
yield cols[1], cols[2].decode('ascii', 'replace')
2556
MTAB_PATH = '/etc/mtab'
2558
class FilesystemFinder(object):
2559
"""Find the filesystem for a particular path."""
2561
def __init__(self, mountpoints):
2564
self._mountpoints = sorted(mountpoints, key=key, reverse=True)
2568
"""Create a FilesystemFinder from an mtab-style file.
2570
Note that this will silenty ignore mtab if it doesn't exist or can not
2573
# TODO(jelmer): Use inotify to be notified when /etc/mtab changes and
2574
# we need to re-read it.
2576
return cls(read_mtab(MTAB_PATH))
2577
except EnvironmentError as e:
2578
trace.mutter('Unable to read mtab: %s', e)
2581
def find(self, path):
2582
"""Find the filesystem used by a particular path.
2584
:param path: Path to find (bytestring or text type)
2585
:return: Filesystem name (as text type) or None, if the filesystem is
2588
for mountpoint, filesystem in self._mountpoints:
2589
if is_inside(mountpoint, path):
2594
_FILESYSTEM_FINDER = None
2597
def get_fs_type(path):
2598
"""Return the filesystem type for the partition a path is in.
2600
:param path: Path to search filesystem type for
2601
:return: A FS type, as string. E.g. "ext2"
2603
global _FILESYSTEM_FINDER
2604
if _FILESYSTEM_FINDER is None:
2605
_FILESYSTEM_FINDER = FilesystemFinder.from_mtab()
2607
if not isinstance(path, bytes):
2608
path = path.encode(_fs_enc)
2610
return _FILESYSTEM_FINDER.find(path)
2613
perf_counter = time.perf_counter
added
removed
breezy/osutils.py
