42
44
from tempfile import mkdtemp
50
from breezy.i18n import gettext
54
from bzrlib.i18n import gettext
57
from bzrlib.symbol_versioning import (
62
warn as warn_deprecated,
53
65
from hashlib import (
72
from bzrlib import symbol_versioning, _fs_enc
75
# Cross platform wall-clock time functionality with decent resolution.
76
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
77
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
78
# synchronized with ``time.time()``, this is only meant to be used to find
79
# delta times by subtracting from another call to this function.
80
timer_func = time.time
81
if sys.platform == 'win32':
82
timer_func = time.clock
66
84
# On win32, O_BINARY is used to indicate the file should
67
85
# be opened in binary mode, rather than text mode.
74
92
O_NOINHERIT = getattr(os, 'O_NOINHERIT', 0)
77
class UnsupportedTimezoneFormat(errors.BzrError):
79
_fmt = ('Unsupported timezone format "%(timezone)s", '
80
'options are "utc", "original", "local".')
82
def __init__(self, timezone):
83
self.timezone = timezone
95
def get_unicode_argv():
97
user_encoding = get_user_encoding()
98
return [a.decode(user_encoding) for a in sys.argv[1:]]
99
except UnicodeDecodeError:
100
raise errors.BzrError(gettext("Parameter {0!r} encoding is unsupported by {1} "
101
"application locale.").format(a, user_encoding))
86
104
def make_readonly(filename):
87
105
"""Make a filename read-only."""
88
106
mod = os.lstat(filename).st_mode
89
107
if not stat.S_ISLNK(mod):
91
109
chmod_if_possible(filename, mod)
94
112
def make_writable(filename):
95
113
mod = os.lstat(filename).st_mode
96
114
if not stat.S_ISLNK(mod):
98
116
chmod_if_possible(filename, mod)
106
124
# It is probably faster to just do the chmod, rather than
107
125
# doing a stat, and then trying to compare
108
126
os.chmod(filename, mode)
109
except (IOError, OSError) as e:
127
except (IOError, OSError),e:
110
128
# Permission/access denied seems to commonly happen on smbfs; there's
111
129
# probably no point warning about it.
112
130
# <https://bugs.launchpad.net/bzr/+bug/606537>
199
213
stat = getattr(os, 'lstat', os.stat)
203
217
if e.errno == errno.ENOENT:
206
raise errors.BzrError(
207
gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
220
raise errors.BzrError(gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
210
223
def fancy_rename(old, new, rename_func, unlink_func):
234
247
file_existed = False
236
249
rename_func(new, tmp_name)
237
except (errors.NoSuchFile,):
250
except (errors.NoSuchFile,), e:
240
253
# RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
241
254
# function raises an IOError with errno is None when a rename fails.
242
255
# This then gets caught here.
243
256
if e.errno not in (None, errno.ENOENT, errno.ENOTDIR):
245
except Exception as e:
246
259
if (getattr(e, 'errno', None) is None
247
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
260
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
250
263
file_existed = True
254
# This may throw an exception, in which case success will
256
rename_func(old, new)
258
except (IOError, OSError) as e:
259
# source and target may be aliases of each other (e.g. on a
260
# case-insensitive filesystem), so we may have accidentally renamed
261
# source by when we tried to rename target
262
if (file_existed and e.errno in (None, errno.ENOENT)
269
# This may throw an exception, in which case success will
271
rename_func(old, new)
273
except (IOError, OSError), e:
274
# source and target may be aliases of each other (e.g. on a
275
# case-insensitive filesystem), so we may have accidentally renamed
276
# source by when we tried to rename target
277
failure_exc = sys.exc_info()
278
if (file_existed and e.errno in (None, errno.ENOENT)
263
279
and old.lower() == new.lower()):
264
# source and target are the same file on a case-insensitive
265
# filesystem, so we don't generate an exception
280
# source and target are the same file on a case-insensitive
281
# filesystem, so we don't generate an exception
271
285
# If the file used to exist, rename it back into place
301
320
# as a special case here by simply removing the first slash, as we consider
302
321
# that breaking POSIX compatibility for this obscure feature is acceptable.
303
322
# This is not a paranoid precaution, as we notably get paths like this when
304
# the repo is hosted at the root of the filesystem, i.e. in "/".
323
# the repo is hosted at the root of the filesystem, i.e. in "/".
305
324
if path.startswith('//'):
329
def _posix_path_from_environ(key):
330
"""Get unicode path from `key` in environment or None if not present
332
Note that posix systems use arbitrary byte strings for filesystem objects,
333
so a path that raises BadFilenameEncoding here may still be accessible.
335
val = os.environ.get(key, None)
339
return val.decode(_fs_enc)
340
except UnicodeDecodeError:
341
# GZ 2011-12-12:Ideally want to include `key` in the exception message
342
raise errors.BadFilenameEncoding(val, _fs_enc)
310
345
def _posix_get_home_dir():
311
346
"""Get the home directory of the current user as a unicode path"""
312
347
path = posixpath.expanduser("~")
314
349
return path.decode(_fs_enc)
315
except AttributeError:
317
350
except UnicodeDecodeError:
318
351
raise errors.BadFilenameEncoding(path, _fs_enc)
321
354
def _posix_getuser_unicode():
322
355
"""Get username from environment or password database as unicode"""
323
return getpass.getuser()
356
name = getpass.getuser()
357
user_encoding = get_user_encoding()
359
return name.decode(user_encoding)
360
except UnicodeDecodeError:
361
raise errors.BzrError("Encoding of username %r is unsupported by %s "
362
"application locale." % (name, user_encoding))
326
365
def _win32_fixdrive(path):
339
378
def _win32_abspath(path):
340
379
# Real ntpath.abspath doesn't have a problem with a unicode cwd
341
return _win32_fixdrive(ntpath.abspath(path).replace('\\', '/'))
380
return _win32_fixdrive(ntpath.abspath(unicode(path)).replace('\\', '/'))
383
def _win98_abspath(path):
384
"""Return the absolute version of a path.
385
Windows 98 safe implementation (python reimplementation
386
of Win32 API function GetFullPathNameW)
391
# \\HOST\path => //HOST/path
392
# //HOST/path => //HOST/path
393
# path => C:/cwd/path
396
# check for absolute path
397
drive = ntpath.splitdrive(path)[0]
398
if drive == '' and path[:2] not in('//','\\\\'):
400
# we cannot simply os.path.join cwd and path
401
# because os.path.join('C:','/path') produce '/path'
402
# and this is incorrect
403
if path[:1] in ('/','\\'):
404
cwd = ntpath.splitdrive(cwd)[0]
406
path = cwd + '\\' + path
407
return _win32_fixdrive(ntpath.normpath(path).replace('\\', '/'))
344
410
def _win32_realpath(path):
345
411
# Real ntpath.realpath doesn't have a problem with a unicode cwd
346
return _win32_fixdrive(ntpath.realpath(path).replace('\\', '/'))
412
return _win32_fixdrive(ntpath.realpath(unicode(path)).replace('\\', '/'))
349
415
def _win32_pathjoin(*args):
353
419
def _win32_normpath(path):
354
return _win32_fixdrive(ntpath.normpath(path).replace('\\', '/'))
420
return _win32_fixdrive(ntpath.normpath(unicode(path)).replace('\\', '/'))
357
423
def _win32_getcwd():
358
return _win32_fixdrive(_getcwd().replace('\\', '/'))
424
return _win32_fixdrive(os.getcwdu().replace('\\', '/'))
361
427
def _win32_mkdtemp(*args, **kwargs):
372
438
fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
374
440
if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY, errno.EINVAL):
375
441
# If we try to rename a non-existant file onto cwd, we get
376
442
# EPERM or EACCES instead of ENOENT, this will raise ENOENT
394
460
def _rename_wrapper(old, new):
396
462
rename_func(old, new)
398
464
detailed_error = OSError(e.errno, e.strerror +
399
" [occurred when renaming '%s' to '%s']" %
465
" [occurred when renaming '%s' to '%s']" %
401
467
detailed_error.old_filename = old
402
468
detailed_error.new_filename = new
403
469
raise detailed_error
405
471
return _rename_wrapper
411
473
# Default rename wraps os.rename()
412
474
rename = _rename_wrap_exception(os.rename)
518
587
output_encoding = get_user_encoding()
520
589
mutter('cp0 is invalid encoding.'
521
' encoding stdout as osutils.get_user_encoding() %r',
590
' encoding stdout as osutils.get_user_encoding() %r',
525
594
codecs.lookup(output_encoding)
526
595
except LookupError:
527
sys.stderr.write('brz: warning:'
596
sys.stderr.write('bzr: warning:'
528
597
' unknown terminal encoding %s.\n'
529
598
' Using encoding %s instead.\n'
530
599
% (output_encoding, get_user_encoding())
532
601
output_encoding = get_user_encoding()
534
603
return output_encoding
672
735
# writes fail on some platforms (e.g. Windows with SMB mounted
674
737
if not segment_size:
675
segment_size = 5242880 # 5MB
676
offsets = range(0, len(bytes), segment_size)
677
view = memoryview(bytes)
738
segment_size = 5242880 # 5MB
739
segments = range(len(bytes) / segment_size + 1)
678
740
write = file_handle.write
679
for offset in offsets:
680
write(view[offset:offset + segment_size])
741
for segment_index in segments:
742
segment = buffer(bytes, segment_index * segment_size, segment_size)
683
746
def file_iterator(input_file, readsize=32768):
744
801
def sha_strings(strings, _factory=sha):
745
802
"""Return the sha-1 of concatenation of strings"""
747
for string in strings:
804
map(s.update, strings)
752
808
def sha_string(f, _factory=sha):
753
# GZ 2017-09-16: Dodgy if factory is ever not sha, probably shouldn't be.
754
return _hexdigest(_factory(f))
809
return _factory(f).hexdigest()
757
812
def fingerprint_file(f):
759
814
return {'size': len(b),
760
'sha1': _hexdigest(sha(b))}
815
'sha1': sha(b).hexdigest()}
763
818
def compare_files(a, b):
797
851
:param show_offset: Whether to append the timezone.
799
853
(date_fmt, tt, offset_str) = \
800
_format_date(t, offset, timezone, date_fmt, show_offset)
854
_format_date(t, offset, timezone, date_fmt, show_offset)
801
855
date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
802
856
date_str = time.strftime(date_fmt, tt)
803
857
return date_str + offset_str
841
895
:param show_offset: Whether to append the timezone.
843
897
(date_fmt, tt, offset_str) = \
844
_format_date(t, offset, timezone, date_fmt, show_offset)
898
_format_date(t, offset, timezone, date_fmt, show_offset)
845
899
date_str = time.strftime(date_fmt, tt)
846
if not isinstance(date_str, str):
900
if not isinstance(date_str, unicode):
847
901
date_str = date_str.decode(get_user_encoding(), 'replace')
848
902
return date_str + offset_str
902
956
plural_seconds = ''
904
958
plural_seconds = 's'
905
if minutes < 90: # print minutes, seconds up to 90 minutes
959
if minutes < 90: # print minutes, seconds up to 90 minutes
907
961
return '%d minute, %d second%s %s' % (
908
minutes, seconds, plural_seconds, direction)
962
minutes, seconds, plural_seconds, direction)
910
964
return '%d minutes, %d second%s %s' % (
911
minutes, seconds, plural_seconds, direction)
965
minutes, seconds, plural_seconds, direction)
913
967
hours = int(minutes / 60)
914
968
minutes -= 60 * hours
923
977
return '%d hours, %d minute%s %s' % (hours, minutes,
924
978
plural_minutes, direction)
928
981
"""Return size of given open file."""
929
982
return os.fstat(f.fileno())[stat.ST_SIZE]
932
# Alias os.urandom to support platforms (which?) without /dev/urandom and
985
# Alias os.urandom to support platforms (which?) without /dev/urandom and
933
986
# override if it doesn't work. Avoid checking on windows where there is
934
987
# significant initialisation cost that can be avoided for some bzr calls.
962
1013
for raw_byte in rand_bytes(num):
963
s += ALNUM[raw_byte % 36]
1014
s += ALNUM[ord(raw_byte) % 36]
967
# TODO: We could later have path objects that remember their list
968
# decomposition (might be too tricksy though.)
1018
## TODO: We could later have path objects that remember their list
1019
## decomposition (might be too tricksy though.)
970
1021
def splitpath(p):
971
1022
"""Turn string into list of parts."""
972
use_bytes = isinstance(p, bytes)
973
if os.path.sep == '\\':
974
# split on either delimiter because people might use either on
977
ps = re.split(b'[\\\\/]', p)
979
ps = re.split(r'[\\/]', p)
988
current_empty_dir = (b'.', b'')
991
current_empty_dir = ('.', '')
1023
# split on either delimiter because people might use either on
1025
ps = re.split(r'[\\/]', p)
996
1030
raise errors.BzrError(gettext("sorry, %r not allowed in path") % f)
997
elif f in current_empty_dir:
1031
elif (f == '.') or (f == ''):
1032
1066
implementation should be loaded instead::
1035
>>> import breezy._fictional_extension_pyx
1069
>>> import bzrlib._fictional_extension_pyx
1036
1070
>>> except ImportError, e:
1037
>>> breezy.osutils.failed_to_load_extension(e)
1038
>>> import breezy._fictional_extension_py
1071
>>> bzrlib.osutils.failed_to_load_extension(e)
1072
>>> import bzrlib._fictional_extension_py
1040
1074
# NB: This docstring is just an example, not a doctest, because doctest
1041
1075
# currently can't cope with the use of lazy imports in this namespace --
1054
1088
def report_extension_load_failures():
1055
1089
if not _extension_load_failures:
1057
if config.GlobalConfig().suppress_warning('missing_extensions'):
1091
if config.GlobalStack().get('ignore_missing_extensions'):
1059
1093
# the warnings framework should by default show this only once
1060
from .trace import warning
1094
from bzrlib.trace import warning
1062
"brz: warning: some compiled extensions could not be loaded; "
1063
"see ``brz help missing-extensions``")
1096
"bzr: warning: some compiled extensions could not be loaded; "
1097
"see <https://answers.launchpad.net/bzr/+faq/703>")
1064
1098
# we no longer show the specific missing extensions here, because it makes
1065
1099
# the message too long and scary - see
1066
1100
# https://bugs.launchpad.net/bzr/+bug/430529
1070
from ._chunks_to_lines_pyx import chunks_to_lines
1071
except ImportError as e:
1104
from bzrlib._chunks_to_lines_pyx import chunks_to_lines
1105
except ImportError, e:
1072
1106
failed_to_load_extension(e)
1073
from ._chunks_to_lines_py import chunks_to_lines
1107
from bzrlib._chunks_to_lines_py import chunks_to_lines
1076
1110
def split_lines(s):
1077
1111
"""Split s into lines, but without removing the newline characters."""
1078
1112
# Trivially convert a fulltext into a 'chunked' representation, and let
1079
1113
# chunks_to_lines do the heavy lifting.
1080
if isinstance(s, bytes):
1114
if isinstance(s, str):
1081
1115
# chunks_to_lines only supports 8-bit strings
1082
1116
return chunks_to_lines([s])
1120
1153
Will delete even if readonly.
1123
_delete_file_or_dir(path)
1124
except (OSError, IOError) as e:
1156
_delete_file_or_dir(path)
1157
except (OSError, IOError), e:
1125
1158
if e.errno in (errno.EPERM, errno.EACCES):
1126
1159
# make writable and try again
1188
1221
# 3) '\xa0' isn't unicode safe since it is >128.
1190
if isinstance(s, str):
1193
ws = (b' ', b'\t', b'\n', b'\r', b'\v', b'\f')
1223
# This should *not* be a unicode set of characters in case the source
1224
# string is not a Unicode string. We can auto-up-cast the characters since
1225
# they are ascii, but we don't want to auto-up-cast the string in case it
1227
for ch in ' \t\n\r\v\f':
1268
1301
abs_base = abspath(base)
1269
1302
current = abs_base
1303
_listdir = os.listdir
1271
1305
# use an explicit iterator so we can easily consume the rest on early exit.
1272
1306
bit_iter = iter(rel.split('/'))
1273
1307
for bit in bit_iter:
1274
1308
lbit = bit.lower()
1276
next_entries = scandir(current)
1277
except OSError: # enoent, eperm, etc
1310
next_entries = _listdir(current)
1311
except OSError: # enoent, eperm, etc
1278
1312
# We can't find this in the filesystem, so just append the
1279
1313
# remaining bits.
1280
1314
current = pathjoin(current, bit, *list(bit_iter))
1282
for entry in next_entries:
1283
if lbit == entry.name.lower():
1284
current = entry.path
1316
for look in next_entries:
1317
if lbit == look.lower():
1318
current = pathjoin(current, look)
1287
1321
# got to the end, nothing matched, so we just return the
1335
1367
Otherwise it is decoded from utf-8. If decoding fails, the exception is
1336
1368
wrapped in a BzrBadParameterNotUnicode exception.
1338
if isinstance(unicode_or_utf8_string, str):
1370
if isinstance(unicode_or_utf8_string, unicode):
1339
1371
return unicode_or_utf8_string
1341
1373
return unicode_or_utf8_string.decode('utf8')
1349
1381
If it is a str, it is returned.
1350
1382
If it is Unicode, it is encoded into a utf-8 string.
1352
if isinstance(unicode_or_utf8_string, bytes):
1384
if isinstance(unicode_or_utf8_string, str):
1353
1385
# TODO: jam 20070209 This is overkill, and probably has an impact on
1354
1386
# performance if we are dealing with lots of apis that want a
1355
1387
# utf-8 revision id
1362
1394
return unicode_or_utf8_string.encode('utf-8')
1397
_revision_id_warning = ('Unicode revision ids were deprecated in bzr 0.15.'
1398
' Revision id generators should be creating utf8'
1402
def safe_revision_id(unicode_or_utf8_string, warn=True):
1403
"""Revision ids should now be utf8, but at one point they were unicode.
1405
:param unicode_or_utf8_string: A possibly Unicode revision_id. (can also be
1407
:param warn: Functions that are sanitizing user data can set warn=False
1408
:return: None or a utf8 revision id.
1410
if (unicode_or_utf8_string is None
1411
or unicode_or_utf8_string.__class__ == str):
1412
return unicode_or_utf8_string
1414
symbol_versioning.warn(_revision_id_warning, DeprecationWarning,
1416
return cache_utf8.encode(unicode_or_utf8_string)
1419
_file_id_warning = ('Unicode file ids were deprecated in bzr 0.15. File id'
1420
' generators should be creating utf8 file ids.')
1423
def safe_file_id(unicode_or_utf8_string, warn=True):
1424
"""File ids should now be utf8, but at one point they were unicode.
1426
This is the same as safe_utf8, except it uses the cached encode functions
1427
to save a little bit of performance.
1429
:param unicode_or_utf8_string: A possibly Unicode file_id. (can also be
1431
:param warn: Functions that are sanitizing user data can set warn=False
1432
:return: None or a utf8 file id.
1434
if (unicode_or_utf8_string is None
1435
or unicode_or_utf8_string.__class__ == str):
1436
return unicode_or_utf8_string
1438
symbol_versioning.warn(_file_id_warning, DeprecationWarning,
1440
return cache_utf8.encode(unicode_or_utf8_string)
1365
1443
_platform_normalizes_filenames = False
1366
1444
if sys.platform == 'darwin':
1367
1445
_platform_normalizes_filenames = True
1390
1468
can be accessed by that path.
1393
if isinstance(path, bytes):
1394
path = path.decode(sys.getfilesystemencoding())
1395
return unicodedata.normalize('NFC', path), True
1471
return unicodedata.normalize('NFC', unicode(path)), True
1398
1474
def _inaccessible_normalized_filename(path):
1399
1475
__doc__ = _accessible_normalized_filename.__doc__
1401
if isinstance(path, bytes):
1402
path = path.decode(sys.getfilesystemencoding())
1403
normalized = unicodedata.normalize('NFC', path)
1477
normalized = unicodedata.normalize('NFC', unicode(path))
1404
1478
return normalized, normalized == path
1460
1534
_terminal_size_state = 'no_data'
1461
1535
_first_terminal_size = None
1464
1537
def terminal_width():
1465
1538
"""Return terminal width.
1467
1540
None is returned if the width can't established precisely.
1470
- if BRZ_COLUMNS is set, returns its value
1543
- if BZR_COLUMNS is set, returns its value
1471
1544
- if there is no controlling terminal, returns None
1472
1545
- query the OS, if the queried size has changed since the last query,
1473
1546
return its value,
1488
1561
# Note to implementors: if changing the rules for determining the width,
1489
1562
# make sure you've considered the behaviour in these cases:
1490
1563
# - M-x shell in emacs, where $COLUMNS is set and TIOCGWINSZ returns 0,0.
1491
# - brz log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1564
# - bzr log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1493
1566
# - (add more interesting cases here, if you find any)
1494
1567
# Some programs implement "Use $COLUMNS (if set) until SIGWINCH occurs",
1498
1571
# time so we can notice if the reported size has changed, which should have
1499
1572
# a similar effect.
1501
# If BRZ_COLUMNS is set, take it, user is always right
1574
# If BZR_COLUMNS is set, take it, user is always right
1502
1575
# Except if they specified 0 in which case, impose no limit here
1504
width = int(os.environ['BRZ_COLUMNS'])
1577
width = int(os.environ['BZR_COLUMNS'])
1505
1578
except (KeyError, ValueError):
1507
1580
if width is not None:
1549
1622
def _win32_terminal_size(width, height):
1550
width, height = win32utils.get_console_size(
1551
defaultx=width, defaulty=height)
1623
width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1552
1624
return width, height
1555
1627
def _ioctl_terminal_size(width, height):
1629
import struct, fcntl, termios
1560
1630
s = struct.pack('HHHH', 0, 0, 0, 0)
1561
1631
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
1562
1632
height, width = struct.unpack('HHHH', x)[0:2]
1580
1649
_terminal_size = _ioctl_terminal_size
1583
def supports_executable(path):
1584
"""Return if filesystem at path supports executable bit.
1586
:param path: Path for which to check the file system
1587
:return: boolean indicating whether executable bit can be stored/relied upon
1589
if sys.platform == 'win32':
1592
fs_type = get_fs_type(path)
1593
except errors.DependencyNotPresent as e:
1594
trace.mutter('Unable to get fs type for %r: %s', path, e)
1596
if fs_type in ('vfat', 'ntfs'):
1597
# filesystems known to not support executable bit
1602
def supports_symlinks(path):
1603
"""Return if the filesystem at path supports the creation of symbolic links.
1606
if not has_symlinks():
1609
fs_type = get_fs_type(path)
1610
except errors.DependencyNotPresent as e:
1611
trace.mutter('Unable to get fs type for %r: %s', path, e)
1613
if fs_type in ('vfat', 'ntfs'):
1614
# filesystems known to not support symlinks
1652
def supports_executable():
1653
return sys.platform != "win32"
1619
1656
def supports_posix_readonly():
1660
1699
raise errors.IllegalPath(path)
1663
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1667
scandir = os.scandir
1668
except AttributeError: # Python < 3
1669
lazy_import(globals(), """\
1670
from scandir import scandir
1702
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1674
1704
def _is_error_enotdir(e):
1675
1705
"""Check if this exception represents ENOTDIR.
1687
1717
:return: True if this represents an ENOTDIR error. False otherwise.
1689
1719
en = getattr(e, 'errno', None)
1690
if (en == errno.ENOTDIR or
1691
(sys.platform == 'win32' and
1692
(en == _WIN32_ERROR_DIRECTORY or
1694
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1720
if (en == errno.ENOTDIR
1721
or (sys.platform == 'win32'
1722
and (en == _WIN32_ERROR_DIRECTORY
1723
or (en == errno.EINVAL
1724
and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1725
1755
rooted higher up.
1726
1756
:return: an iterator over the dirs.
1728
# TODO there is a bit of a smell where the results of the directory-
1758
#TODO there is a bit of a smell where the results of the directory-
1729
1759
# summary in this, and the path from the root, may not agree
1730
1760
# depending on top and prefix - i.e. ./foo and foo as a pair leads to
1731
1761
# potentially confusing output. We should make this more robust - but
1732
1762
# not at a speed cost. RBC 20060731
1733
1764
_directory = _directory_kind
1765
_listdir = os.listdir
1766
_kind_from_mode = file_kind_from_stat_mode
1734
1767
pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
1736
1769
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1742
1775
top_slash = top + u'/'
1778
append = dirblock.append
1746
for entry in scandir(top):
1747
name = decode_filename(entry.name)
1748
statvalue = entry.stat(follow_symlinks=False)
1749
kind = file_kind_from_stat_mode(statvalue.st_mode)
1750
dirblock.append((relprefix + name, name, kind, statvalue, entry.path))
1751
except OSError as e:
1780
names = sorted(map(decode_filename, _listdir(top)))
1752
1782
if not _is_error_enotdir(e):
1754
except UnicodeDecodeError as e:
1755
raise errors.BadFilenameEncoding(e.object, _fs_enc)
1786
abspath = top_slash + name
1787
statvalue = _lstat(abspath)
1788
kind = _kind_from_mode(statvalue.st_mode)
1789
append((relprefix + name, name, kind, statvalue, abspath))
1757
1790
yield (relroot, top), dirblock
1759
1792
# push the user specified dirs from dirblock
1805
1838
global _selected_dir_reader
1806
1839
if _selected_dir_reader is None:
1807
if sys.platform == "win32":
1840
if sys.platform == "win32" and win32utils.winver == 'Windows NT':
1841
# Win98 doesn't have unicode apis like FindFirstFileW
1842
# TODO: We possibly could support Win98 by falling back to the
1843
# original FindFirstFile, and using TCHAR instead of WCHAR,
1844
# but that gets a bit tricky, and requires custom compiling
1809
from ._walkdirs_win32 import Win32ReadDir
1847
from bzrlib._walkdirs_win32 import Win32ReadDir
1810
1848
_selected_dir_reader = Win32ReadDir()
1811
1849
except ImportError:
1813
1851
elif _fs_enc in ('utf-8', 'ascii'):
1815
from ._readdir_pyx import UTF8DirReader
1853
from bzrlib._readdir_pyx import UTF8DirReader
1816
1854
_selected_dir_reader = UTF8DirReader()
1817
except ImportError as e:
1855
except ImportError, e:
1818
1856
failed_to_load_extension(e)
1865
1903
See DirReader.read_dir for details.
1867
1905
_utf8_encode = self._utf8_encode
1869
def _fs_decode(s): return s.decode(_fs_enc)
1871
def _fs_encode(s): return s.encode(_fs_enc)
1907
_listdir = os.listdir
1908
_kind_from_mode = file_kind_from_stat_mode
1874
relprefix = prefix + b'/'
1911
relprefix = prefix + '/'
1877
top_slash = top + '/'
1914
top_slash = top + u'/'
1880
1917
append = dirblock.append
1881
for entry in scandir(safe_utf8(top)):
1918
for name in sorted(_listdir(top)):
1883
name = _fs_decode(entry.name)
1920
name_utf8 = _utf8_encode(name)[0]
1884
1921
except UnicodeDecodeError:
1885
1922
raise errors.BadFilenameEncoding(
1886
relprefix + entry.name, _fs_enc)
1923
_utf8_encode(relprefix)[0] + name, _fs_enc)
1887
1924
abspath = top_slash + name
1888
name_utf8 = _utf8_encode(name)[0]
1889
statvalue = entry.stat(follow_symlinks=False)
1890
kind = file_kind_from_stat_mode(statvalue.st_mode)
1925
statvalue = _lstat(abspath)
1926
kind = _kind_from_mode(statvalue.st_mode)
1891
1927
append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
1892
return sorted(dirblock)
1895
1931
def copy_tree(from_path, to_path, handlers={}):
1965
2001
This can be used to sort paths in the same way that walkdirs does.
1967
return (dirname(path), path)
2003
return (dirname(path) , path)
1970
2006
def compare_paths_prefix_order(path_a, path_b):
1971
2007
"""Compare path_a and path_b to generate the same order walkdirs uses."""
1972
2008
key_a = path_prefix_key(path_a)
1973
2009
key_b = path_prefix_key(path_b)
1974
return (key_a > key_b) - (key_a < key_b)
2010
return cmp(key_a, key_b)
1977
2013
_cached_user_encoding = None
1980
def get_user_encoding():
2016
def get_user_encoding(use_cache=DEPRECATED_PARAMETER):
1981
2017
"""Find out what the preferred user encoding is.
1983
2019
This is generally the encoding that is used for command line parameters
1987
2023
:return: A string defining the preferred user encoding
1989
2025
global _cached_user_encoding
2026
if deprecated_passed(use_cache):
2027
warn_deprecated("use_cache should only have been used for tests",
2028
DeprecationWarning, stacklevel=2)
1990
2029
if _cached_user_encoding is not None:
1991
2030
return _cached_user_encoding
2058
2097
def read_bytes_from_socket(sock, report_activity=None,
2059
max_read_size=MAX_SOCKET_CHUNK):
2098
max_read_size=MAX_SOCKET_CHUNK):
2060
2099
"""Read up to max_read_size of bytes from sock and notify of progress.
2062
2101
Translates "Connection reset by peer" into file-like EOF (return an
2063
2102
empty string rather than raise an error), and repeats the recv if
2064
2103
interrupted by a signal.
2068
data = sock.recv(max_read_size)
2069
except socket.error as e:
2107
bytes = sock.recv(max_read_size)
2108
except socket.error, e:
2070
2109
eno = e.args[0]
2071
2110
if eno in _end_of_stream_errors:
2072
2111
# The connection was closed by the other side. Callers expect
2073
2112
# an empty string to signal end-of-stream.
2075
2114
elif eno == errno.EINTR:
2076
2115
# Retry the interrupted recv.
2080
2119
if report_activity is not None:
2081
report_activity(len(data), 'read')
2120
report_activity(len(bytes), 'read')
2085
2124
def recv_all(socket, count):
2118
2157
byte_count = len(bytes)
2119
view = memoryview(bytes)
2120
2158
while sent_total < byte_count:
2122
sent = sock.send(view[sent_total:sent_total + MAX_SOCKET_CHUNK])
2123
except (socket.error, IOError) as e:
2160
sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))
2161
except (socket.error, IOError), e:
2124
2162
if e.args[0] in _end_of_stream_errors:
2125
2163
raise errors.ConnectionReset(
2126
2164
"Error trying to write to socket", e)
2190
2227
If and when pkg_resources becomes a standard library, this routine
2191
2228
can delegate to it.
2193
# Check package name is within breezy
2194
if package == "breezy":
2230
# Check package name is within bzrlib
2231
if package == "bzrlib":
2195
2232
resource_relpath = resource_name
2196
elif package.startswith("breezy."):
2197
package = package[len("breezy."):].replace('.', os.sep)
2233
elif package.startswith("bzrlib."):
2234
package = package[len("bzrlib."):].replace('.', os.sep)
2198
2235
resource_relpath = pathjoin(package, resource_name)
2200
raise errors.BzrError('resource package %s not in breezy' % package)
2237
raise errors.BzrError('resource package %s not in bzrlib' % package)
2202
2239
# Map the resource to a file and read its contents
2203
base = dirname(breezy.__file__)
2240
base = dirname(bzrlib.__file__)
2204
2241
if getattr(sys, 'frozen', None): # bzr.exe
2205
2242
base = abspath(pathjoin(base, '..', '..'))
2206
with open(pathjoin(base, resource_relpath), "rt") as f:
2243
f = file(pathjoin(base, resource_relpath), "rU")
2207
2245
return f.read()
2210
2249
def file_kind_from_stat_mode_thunk(mode):
2211
2250
global file_kind_from_stat_mode
2212
2251
if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
2214
from ._readdir_pyx import UTF8DirReader
2253
from bzrlib._readdir_pyx import UTF8DirReader
2215
2254
file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
2255
except ImportError, e:
2217
2256
# This is one time where we won't warn that an extension failed to
2218
2257
# load. The extension is never available on Windows anyway.
2219
from ._readdir_py import (
2258
from bzrlib._readdir_py import (
2220
2259
_kind_from_mode as file_kind_from_stat_mode
2222
2261
return file_kind_from_stat_mode(mode)
2225
2262
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2228
2264
def file_stat(f, _lstat=os.lstat):
2231
2267
return _lstat(f)
2232
except OSError as e:
2233
2269
if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
2234
2270
raise errors.NoSuchFile(f)
2238
2273
def file_kind(f, _lstat=os.lstat):
2239
2274
stat_value = file_stat(f, _lstat)
2240
2275
return file_kind_from_stat_mode(stat_value.st_mode)
2243
2277
def until_no_eintr(f, *a, **kw):
2244
2278
"""Run f(*a, **kw), retrying if an EINTR error occurs.
2250
2284
Keep in mind that this is not a complete solution to EINTR. There is
2251
2285
probably code in the Python standard library and other dependencies that
2252
2286
may encounter EINTR if a signal arrives (and there is signal handler for
2253
that signal). So this function can reduce the impact for IO that breezy
2287
that signal). So this function can reduce the impact for IO that bzrlib
2254
2288
directly controls, but it is not a complete solution.
2256
2290
# Borrowed from Twisted's twisted.python.util.untilConcludes function.
2259
2293
return f(*a, **kw)
2260
except (IOError, OSError) as e:
2294
except (IOError, OSError), e:
2261
2295
if e.errno == errno.EINTR:
2300
@deprecated_function(deprecated_in((2, 2, 0)))
2301
def re_compile_checked(re_string, flags=0, where=""):
2302
"""Return a compiled re, or raise a sensible error.
2304
This should only be used when compiling user-supplied REs.
2306
:param re_string: Text form of regular expression.
2307
:param flags: eg re.IGNORECASE
2308
:param where: Message explaining to the user the context where
2309
it occurred, eg 'log search filter'.
2311
# from https://bugs.launchpad.net/bzr/+bug/251352
2313
re_obj = re.compile(re_string, flags)
2316
except errors.InvalidPattern, e:
2318
where = ' in ' + where
2319
# despite the name 'error' is a type
2320
raise errors.BzrCommandError('Invalid regular expression%s: %s'
2266
2324
if sys.platform == "win32":
2296
2354
stdout=subprocess.PIPE).communicate()[0]
2297
2355
elif sys.platform == 'sunos5':
2298
2356
def _local_concurrency():
2299
return subprocess.Popen(['psrinfo', '-p', ],
2357
return subprocess.Popen(['psrinfo', '-p',],
2300
2358
stdout=subprocess.PIPE).communicate()[0]
2301
2359
elif sys.platform == "win32":
2302
2360
def _local_concurrency():
2322
2379
if _cached_local_concurrency is not None and use_cache:
2323
2380
return _cached_local_concurrency
2325
concurrency = os.environ.get('BRZ_CONCURRENCY', None)
2382
concurrency = os.environ.get('BZR_CONCURRENCY', None)
2326
2383
if concurrency is None:
2327
import multiprocessing
2385
import multiprocessing
2329
2386
concurrency = multiprocessing.cpu_count()
2330
except NotImplementedError:
2331
# multiprocessing.cpu_count() isn't implemented on all platforms
2387
except (ImportError, NotImplementedError):
2388
# multiprocessing is only available on Python >= 2.6
2389
# and multiprocessing.cpu_count() isn't implemented on all
2333
2392
concurrency = _local_concurrency()
2334
2393
except (OSError, IOError):
2350
2409
self.encode = encode
2352
2411
def write(self, object):
2353
if isinstance(object, str):
2412
if type(object) is str:
2354
2413
self.stream.write(object)
2356
2415
data, _ = self.encode(object, self.errors)
2357
2416
self.stream.write(data)
2360
2418
if sys.platform == 'win32':
2361
2419
def open_file(filename, mode='r', bufsize=-1):
2362
2420
"""This function is used to override the ``open`` builtin.
2484
2542
# exists, though not ours
2487
trace.mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2545
mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2488
2546
# Don't really know.
2491
2549
# Exists and our process: not dead.
2495
2552
if sys.platform == "win32":
2496
2553
is_local_pid_dead = win32utils.is_local_pid_dead
2525
2582
def ensure_empty_directory_exists(path, exception_class):
2526
2583
"""Make sure a local directory exists and is empty.
2528
2585
If it does not exist, it is created. If it exists and is not empty, an
2529
2586
instance of exception_class is raised.
2533
except OSError as e:
2534
2591
if e.errno != errno.EEXIST:
2536
2593
if os.listdir(path) != []:
2537
2594
raise exception_class(path)
2540
def read_mtab(path):
2541
"""Read an fstab-style file and extract mountpoint+filesystem information.
2543
:param path: Path to read from
2544
:yield: Tuples with mountpoints (as bytestrings) and filesystem names
2546
with open(path, 'rb') as f:
2548
if line.startswith(b'#'):
2553
yield cols[1], cols[2].decode('ascii', 'replace')
2556
MTAB_PATH = '/etc/mtab'
2558
class FilesystemFinder(object):
2559
"""Find the filesystem for a particular path."""
2561
def __init__(self, mountpoints):
2564
self._mountpoints = sorted(mountpoints, key=key, reverse=True)
2568
"""Create a FilesystemFinder from an mtab-style file.
2570
Note that this will silenty ignore mtab if it doesn't exist or can not
2573
# TODO(jelmer): Use inotify to be notified when /etc/mtab changes and
2574
# we need to re-read it.
2576
return cls(read_mtab(MTAB_PATH))
2577
except EnvironmentError as e:
2578
trace.mutter('Unable to read mtab: %s', e)
2581
def find(self, path):
2582
"""Find the filesystem used by a particular path.
2584
:param path: Path to find (bytestring or text type)
2585
:return: Filesystem name (as text type) or None, if the filesystem is
2588
for mountpoint, filesystem in self._mountpoints:
2589
if is_inside(mountpoint, path):
2594
_FILESYSTEM_FINDER = None
2597
def get_fs_type(path):
2598
"""Return the filesystem type for the partition a path is in.
2600
:param path: Path to search filesystem type for
2601
:return: A FS type, as string. E.g. "ext2"
2603
global _FILESYSTEM_FINDER
2604
if _FILESYSTEM_FINDER is None:
2605
_FILESYSTEM_FINDER = FilesystemFinder.from_mtab()
2607
if not isinstance(path, bytes):
2608
path = path.encode(_fs_enc)
2610
return _FILESYSTEM_FINDER.find(path)
2613
perf_counter = time.perf_counter
2597
def is_environment_error(evalue):
2598
"""True if exception instance is due to a process environment issue
2600
This includes OSError and IOError, but also other errors that come from
2601
the operating system or core libraries but are not subclasses of those.
2603
if isinstance(evalue, (EnvironmentError, select.error)):
2605
if sys.platform == "win32" and win32utils._is_pywintypes_error(evalue):
added
removed
bzrlib/osutils.py
