/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5594.1.1 by Vincent Ladeuil
Fix socketpair-based SSH transport leaking socket into other child processes
1
# Copyright (C) 2005-2011 Canonical Ltd
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
2
#
1 by mbp at sourcefrog
import from baz patch-364
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
7
#
1 by mbp at sourcefrog
import from baz patch-364
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
12
#
1 by mbp at sourcefrog
import from baz patch-364
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1 by mbp at sourcefrog
import from baz patch-364
16
6379.6.1 by Jelmer Vernooij
Import absolute_import in a few places.
17
from __future__ import absolute_import
18
5225.1.1 by Andrew Bennetts
Don't lazy_import errno in osutils; it's a builtin module, and that lazy_import is implicated in a FreeBSD builder failure.
19
import errno
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
20
import os
21
import re
22
import stat
23
import sys
24
import time
5051.2.1 by Benjamin Peterson
move codecs import out of lazy section since it is used on module import
25
import codecs
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
26
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
27
from .lazy_import import lazy_import
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
28
lazy_import(globals(), """
2215.6.1 by James Henstridge
Don't rely on time.timezone and time.altzone in local_time_offset(),
29
from datetime import datetime
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
30
from datetime import timedelta
5187.2.7 by Parth Malwankar
moved getuser_unicode to osutils.
31
import getpass
6383.1.1 by Martin Packman
Simplify get_user_encoding by avoiding locale hacks and assuming setlocale has been called
32
import locale
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
33
import ntpath
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
34
import posixpath
6336.2.1 by Martin Packman
Add is_environment_error() and switch trace to using it
35
import select
5273.1.11 by Vincent Ladeuil
Really fix and better explain why we need both the module and the symbol.
36
# We need to import both shutil and rmtree as we export the later on posix
37
# and need the former on windows
38
import shutil
5273.1.10 by Vincent Ladeuil
Fixed as per jam's review.
39
from shutil import rmtree
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
40
import socket
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
41
import subprocess
5273.1.11 by Vincent Ladeuil
Really fix and better explain why we need both the module and the symbol.
42
# We need to import both tempfile and mkdtemp as we export the later on posix
43
# and need the former on windows
44
import tempfile
45
from tempfile import mkdtemp
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
46
import unicodedata
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
47
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
48
from breezy import (
6059.4.2 by Vincent Ladeuil
Migrate ignore_missing_extensions to stack-based config.
49
    config,
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
50
    errors,
5011.3.16 by Andrew Bennetts
Merge lp:bzr.
51
    trace,
2245.4.6 by Alexander Belchenko
osutils.py: terminal_width() now use win32utils.get_console_size()
52
    win32utils,
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
53
    )
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
54
from breezy.i18n import gettext
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
55
""")
1 by mbp at sourcefrog
import from baz patch-364
56
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
57
from .sixish import (
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
58
    PY3,
59
    text_type,
60
    )
5011.3.3 by Martin
Reintroduce EINTR handling only for socket object functions and general cleanup
61
5848.2.1 by John Arbash Meinel
Break compatibility with python <2.6.
62
from hashlib import (
63
    md5,
64
    sha1 as sha,
65
    )
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
66
3504.4.1 by John Arbash Meinel
Write an alternative 'walkdirs' implementation that uses win32 apis.
67
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
68
import breezy
6630.1.1 by Jelmer Vernooij
Remove deprecated functionality.
69
from . import _fs_enc
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
70
1 by mbp at sourcefrog
import from baz patch-364
71
4889.2.5 by John Arbash Meinel
Review feedback from Andrew.
72
# Cross platform wall-clock time functionality with decent resolution.
73
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
74
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
75
# synchronized with ``time.time()``, this is only meant to be used to find
76
# delta times by subtracting from another call to this function.
4889.2.1 by John Arbash Meinel
Make -Dhpss log debug information for the server process.
77
timer_func = time.time
78
if sys.platform == 'win32':
79
    timer_func = time.clock
80
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
81
# On win32, O_BINARY is used to indicate the file should
82
# be opened in binary mode, rather than text mode.
83
# On other platforms, O_BINARY doesn't exist, because
84
# they always open in binary mode, so it is okay to
4634.140.12 by INADA Naoki
small clean up.
85
# OR with 0 on those platforms.
86
# O_NOINHERIT and O_TEXT exists only on win32 too.
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
87
O_BINARY = getattr(os, 'O_BINARY', 0)
4634.140.12 by INADA Naoki
small clean up.
88
O_TEXT = getattr(os, 'O_TEXT', 0)
4634.140.1 by INADA Naoki
Avoids child process inherits file handles on win32. by using os.fdopen and os.open with O_NOINHERIT instead of builtin open.
89
O_NOINHERIT = getattr(os, 'O_NOINHERIT', 0)
90
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
91
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
92
def get_unicode_argv():
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
93
    if PY3:
94
        return sys.argv[1:]
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
95
    try:
96
        user_encoding = get_user_encoding()
5274.4.8 by Jason Spashett
Reverse merge -r 5280..5279
97
        return [a.decode(user_encoding) for a in sys.argv[1:]]
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
98
    except UnicodeDecodeError:
6138.3.8 by Jonathan Riddell
more error gettext()ing
99
        raise errors.BzrError(gettext("Parameter {0!r} encoding is unsupported by {1} "
100
            "application locale.").format(a, user_encoding))
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
101
102
1 by mbp at sourcefrog
import from baz patch-364
103
def make_readonly(filename):
104
    """Make a filename read-only."""
2949.6.1 by Alexander Belchenko
windows python has os.lstat
105
    mod = os.lstat(filename).st_mode
2568.1.1 by John Arbash Meinel
(Elliot Murphy) Use os.lstat rather than os.stat for osutils.make_readonly/make_writeable
106
    if not stat.S_ISLNK(mod):
6619.3.14 by Jelmer Vernooij
Convert some octal numbers to new notations.
107
        mod = mod & 0o777555
6015.50.1 by Martin Pool
Use a chmod wrapper to cope with eperm from chmod
108
        chmod_if_possible(filename, mod)
1 by mbp at sourcefrog
import from baz patch-364
109
110
111
def make_writable(filename):
2949.6.1 by Alexander Belchenko
windows python has os.lstat
112
    mod = os.lstat(filename).st_mode
2568.1.1 by John Arbash Meinel
(Elliot Murphy) Use os.lstat rather than os.stat for osutils.make_readonly/make_writeable
113
    if not stat.S_ISLNK(mod):
6619.3.14 by Jelmer Vernooij
Convert some octal numbers to new notations.
114
        mod = mod | 0o200
6015.50.1 by Martin Pool
Use a chmod wrapper to cope with eperm from chmod
115
        chmod_if_possible(filename, mod)
116
117
118
def chmod_if_possible(filename, mode):
119
    # Set file mode if that can be safely done.
120
    # Sometimes even on unix the filesystem won't allow it - see
121
    # https://bugs.launchpad.net/bzr/+bug/606537
122
    try:
123
        # It is probably faster to just do the chmod, rather than
124
        # doing a stat, and then trying to compare
125
        os.chmod(filename, mode)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
126
    except (IOError, OSError) as e:
6015.50.2 by Martin Pool
Also suppress EACCES from chmod
127
        # Permission/access denied seems to commonly happen on smbfs; there's
128
        # probably no point warning about it.
6015.50.1 by Martin Pool
Use a chmod wrapper to cope with eperm from chmod
129
        # <https://bugs.launchpad.net/bzr/+bug/606537>
6015.50.2 by Martin Pool
Also suppress EACCES from chmod
130
        if getattr(e, 'errno') in (errno.EPERM, errno.EACCES):
6015.50.3 by Martin Pool
More correct trace message when squelching chmod errors
131
            trace.mutter("ignore error on chmod of %r: %r" % (
132
                filename, e))
6015.50.1 by Martin Pool
Use a chmod wrapper to cope with eperm from chmod
133
            return
134
        raise
1 by mbp at sourcefrog
import from baz patch-364
135
136
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
137
def minimum_path_selection(paths):
138
    """Return the smallset subset of paths which are outside paths.
139
2843.1.1 by Ian Clatworthy
Faster partial commits by walking less data (Robert Collins)
140
    :param paths: A container (and hence not None) of paths.
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
141
    :return: A set of paths sufficient to include everything in paths via
4325.3.3 by Johan Walles
Add unit test and fix for minimum_path_selection() vs directory names with
142
        is_inside, drawn from the paths parameter.
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
143
    """
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
144
    if len(paths) < 2:
145
        return set(paths)
4325.3.3 by Johan Walles
Add unit test and fix for minimum_path_selection() vs directory names with
146
147
    def sort_key(path):
148
        return path.split('/')
149
    sorted_paths = sorted(list(paths), key=sort_key)
150
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
151
    search_paths = [sorted_paths[0]]
152
    for path in sorted_paths[1:]:
4325.3.2 by Johan Walles
Use a linear algorithm for osutil.minimum_path_selection().
153
        if not is_inside(search_paths[-1], path):
154
            # This path is unique, add it
155
            search_paths.append(path)
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
156
4325.3.2 by Johan Walles
Use a linear algorithm for osutil.minimum_path_selection().
157
    return set(search_paths)
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
158
159
1077 by Martin Pool
- avoid compiling REs at module load time
160
_QUOTE_RE = None
969 by Martin Pool
- Add less-sucky is_within_any
161
162
1 by mbp at sourcefrog
import from baz patch-364
163
def quotefn(f):
779 by Martin Pool
- better quotefn for windows: use doublequotes for strings with
164
    """Return a quoted filename filename
165
166
    This previously used backslash quoting, but that works poorly on
167
    Windows."""
168
    # TODO: I'm not really sure this is the best format either.x
1077 by Martin Pool
- avoid compiling REs at module load time
169
    global _QUOTE_RE
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
170
    if _QUOTE_RE is None:
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
171
        _QUOTE_RE = re.compile(r'([^a-zA-Z0-9.,:/\\_~-])')
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
172
779 by Martin Pool
- better quotefn for windows: use doublequotes for strings with
173
    if _QUOTE_RE.search(f):
174
        return '"' + f + '"'
175
    else:
176
        return f
1 by mbp at sourcefrog
import from baz patch-364
177
178
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
179
_directory_kind = 'directory'
180
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
181
def get_umask():
182
    """Return the current umask"""
183
    # Assume that people aren't messing with the umask while running
184
    # XXX: This is not thread safe, but there is no way to get the
185
    #      umask without setting it
186
    umask = os.umask(0)
187
    os.umask(umask)
188
    return umask
189
190
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
191
_kind_marker_map = {
192
    "file": "",
193
    _directory_kind: "/",
194
    "symlink": "@",
1551.10.30 by Aaron Bentley
Merge from bzr.dev
195
    'tree-reference': '+',
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
196
}
1551.10.30 by Aaron Bentley
Merge from bzr.dev
197
198
488 by Martin Pool
- new helper function kind_marker()
199
def kind_marker(kind):
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
200
    try:
201
        return _kind_marker_map[kind]
202
    except KeyError:
5024.1.2 by John Arbash Meinel
Switch so that all unknown files get an empty marker, rather than failing.
203
        # Slightly faster than using .get(, '') when the common case is that
204
        # kind will be found
205
        return ''
1 by mbp at sourcefrog
import from baz patch-364
206
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
207
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
208
lexists = getattr(os.path, 'lexists', None)
209
if lexists is None:
210
    def lexists(f):
211
        try:
2324.2.2 by Dmitry Vasiliev
Fixed lexists() implementation
212
            stat = getattr(os, 'lstat', os.stat)
213
            stat(f)
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
214
            return True
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
215
        except OSError as e:
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
216
            if e.errno == errno.ENOENT:
217
                return False;
218
            else:
6138.3.8 by Jonathan Riddell
more error gettext()ing
219
                raise errors.BzrError(gettext("lstat/stat of ({0!r}): {1!r}").format(f, e))
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
220
1 by mbp at sourcefrog
import from baz patch-364
221
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
222
def fancy_rename(old, new, rename_func, unlink_func):
223
    """A fancy rename, when you don't have atomic rename.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
224
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
225
    :param old: The old path, to rename from
226
    :param new: The new path, to rename to
227
    :param rename_func: The potentially non-atomic rename function
4935.1.1 by Vincent Ladeuil
Support Unicode paths for ftp transport (encoded as utf8).
228
    :param unlink_func: A way to delete the target file if the full rename
229
        succeeds
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
230
    """
231
    # sftp rename doesn't allow overwriting, so play tricks:
232
    base = os.path.basename(new)
233
    dirname = os.path.dirname(new)
4935.1.3 by Vincent Ladeuil
Better fix for fancy_rename respecting callers file encoding.
234
    # callers use different encodings for the paths so the following MUST
235
    # respect that. We rely on python upcasting to unicode if new is unicode
236
    # and keeping a str if not.
237
    tmp_name = 'tmp.%s.%.9f.%d.%s' % (base, time.time(),
238
                                      os.getpid(), rand_chars(10))
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
239
    tmp_name = pathjoin(dirname, tmp_name)
240
241
    # Rename the file out of the way, but keep track if it didn't exist
242
    # We don't want to grab just any exception
243
    # something like EACCES should prevent us from continuing
244
    # The downside is that the rename_func has to throw an exception
245
    # with an errno = ENOENT, or NoSuchFile
246
    file_existed = False
247
    try:
248
        rename_func(new, tmp_name)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
249
    except (errors.NoSuchFile,) as e:
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
250
        pass
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
251
    except IOError as e:
1532 by Robert Collins
Merge in John Meinels integration branch.
252
        # RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
253
        # function raises an IOError with errno is None when a rename fails.
1532 by Robert Collins
Merge in John Meinels integration branch.
254
        # This then gets caught here.
1185.50.37 by John Arbash Meinel
Fixed exception handling for fancy_rename
255
        if e.errno not in (None, errno.ENOENT, errno.ENOTDIR):
1532 by Robert Collins
Merge in John Meinels integration branch.
256
            raise
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
257
    except Exception as e:
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
258
        if (getattr(e, 'errno', None) is None
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
259
            or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
260
            raise
261
    else:
262
        file_existed = True
263
264
    success = False
265
    try:
6619.3.27 by Martin
Unify try/except/finally block to avoid reraise complexity
266
        # This may throw an exception, in which case success will
267
        # not be set.
268
        rename_func(old, new)
269
        success = True
270
    except (IOError, OSError) as e:
271
        # source and target may be aliases of each other (e.g. on a
272
        # case-insensitive filesystem), so we may have accidentally renamed
273
        # source by when we tried to rename target
274
        if (file_existed and e.errno in (None, errno.ENOENT)
275
            and old.lower() == new.lower()):
276
            # source and target are the same file on a case-insensitive
277
            # filesystem, so we don't generate an exception
278
            pass
279
        else:
280
            raise
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
281
    finally:
282
        if file_existed:
283
            # If the file used to exist, rename it back into place
284
            # otherwise just delete it from the tmp location
285
            if success:
1551.15.4 by Aaron Bentley
Revert now-unnecessary changes
286
                unlink_func(tmp_name)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
287
            else:
1185.31.49 by John Arbash Meinel
Some corrections using the new osutils.rename. **ALL TESTS PASS**
288
                rename_func(tmp_name, new)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
289
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
290
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
291
# In Python 2.4.2 and older, os.path.abspath and os.path.realpath
292
# choke on a Unicode string containing a relative path if
293
# os.getcwd() returns a non-sys.getdefaultencoding()-encoded
294
# string.
295
def _posix_abspath(path):
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
296
    # jam 20060426 rather than encoding to fsencoding
297
    # copy posixpath.abspath, but use os.getcwdu instead
298
    if not posixpath.isabs(path):
299
        path = posixpath.join(getcwd(), path)
6015.39.2 by Florian Vichot
Fixed an infinite loop when creating a repo at the root of the filesystem,
300
    return _posix_normpath(path)
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
301
302
303
def _posix_realpath(path):
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
304
    return posixpath.realpath(path.encode(_fs_enc)).decode(_fs_enc)
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
305
306
6015.39.2 by Florian Vichot
Fixed an infinite loop when creating a repo at the root of the filesystem,
307
def _posix_normpath(path):
308
    path = posixpath.normpath(path)
309
    # Bug 861008: posixpath.normpath() returns a path normalized according to
310
    # the POSIX standard, which stipulates (for compatibility reasons) that two
311
    # leading slashes must not be simplified to one, and only if there are 3 or
312
    # more should they be simplified as one. So we treat the leading 2 slashes
313
    # as a special case here by simply removing the first slash, as we consider
314
    # that breaking POSIX compatibility for this obscure feature is acceptable.
315
    # This is not a paranoid precaution, as we notably get paths like this when
316
    # the repo is hosted at the root of the filesystem, i.e. in "/".    
317
    if path.startswith('//'):
318
        path = path[1:]
319
    return path
320
321
6362.2.1 by Martin Packman
Add path_from_environ function for getting unicode paths from envvars
322
def _posix_path_from_environ(key):
323
    """Get unicode path from `key` in environment or None if not present
324
325
    Note that posix systems use arbitrary byte strings for filesystem objects,
326
    so a path that raises BadFilenameEncoding here may still be accessible.
327
    """
328
    val = os.environ.get(key, None)
6651.4.1 by Martin
Rewrite of the plugin module for Python 3 compat and general sanity
329
    if PY3 or val is None:
6362.2.1 by Martin Packman
Add path_from_environ function for getting unicode paths from envvars
330
        return val
331
    try:
332
        return val.decode(_fs_enc)
333
    except UnicodeDecodeError:
334
        # GZ 2011-12-12:Ideally want to include `key` in the exception message
335
        raise errors.BadFilenameEncoding(val, _fs_enc)
336
337
6437.26.1 by Martin Packman
Add and test osutils._get_home_dir for unicode access to home location across platforms
338
def _posix_get_home_dir():
339
    """Get the home directory of the current user as a unicode path"""
340
    path = posixpath.expanduser("~")
341
    try:
342
        return path.decode(_fs_enc)
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
343
    except AttributeError:
344
        return path
6437.26.1 by Martin Packman
Add and test osutils._get_home_dir for unicode access to home location across platforms
345
    except UnicodeDecodeError:
346
        raise errors.BadFilenameEncoding(path, _fs_enc)
347
348
6362.2.4 by Martin Packman
Use native functions for getting username avoiding bug 660174 entirely
349
def _posix_getuser_unicode():
350
    """Get username from environment or password database as unicode"""
351
    name = getpass.getuser()
352
    user_encoding = get_user_encoding()
353
    try:
354
        return name.decode(user_encoding)
355
    except UnicodeDecodeError:
356
        raise errors.BzrError("Encoding of username %r is unsupported by %s "
357
            "application locale." % (name, user_encoding))
358
359
1711.5.2 by John Arbash Meinel
win32 likes to return lowercase drive letters sometimes, and uppercase at other times. normalize this
360
def _win32_fixdrive(path):
361
    """Force drive letters to be consistent.
362
363
    win32 is inconsistent whether it returns lower or upper case
364
    and even if it was consistent the user might type the other
365
    so we force it to uppercase
366
    running python.exe under cmd.exe return capital C:\\
367
    running win32 python inside a cygwin shell returns lowercase c:\\
368
    """
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
369
    drive, path = ntpath.splitdrive(path)
1711.5.2 by John Arbash Meinel
win32 likes to return lowercase drive letters sometimes, and uppercase at other times. normalize this
370
    return drive.upper() + path
371
372
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
373
def _win32_abspath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
374
    # Real ntpath.abspath doesn't have a problem with a unicode cwd
375
    return _win32_fixdrive(ntpath.abspath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
376
377
378
def _win32_realpath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
379
    # Real ntpath.realpath doesn't have a problem with a unicode cwd
380
    return _win32_fixdrive(ntpath.realpath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
381
382
383
def _win32_pathjoin(*args):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
384
    return ntpath.join(*args).replace('\\', '/')
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
385
386
387
def _win32_normpath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
388
    return _win32_fixdrive(ntpath.normpath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
389
390
391
def _win32_getcwd():
6619.3.26 by Martin
Fix fallout from 2to3 getcwdu transformation and other test uses
392
    return _win32_fixdrive(_getcwd().replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
393
394
395
def _win32_mkdtemp(*args, **kwargs):
5331.3.1 by Martin
Avoid infinite recursion with _win32_mkdtemp by using module namespaced name
396
    return _win32_fixdrive(tempfile.mkdtemp(*args, **kwargs).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
397
398
399
def _win32_rename(old, new):
1711.7.6 by John Arbash Meinel
Change _win32_rename() so that it raises ENOENT *before* it tries any renaming.
400
    """We expect to be able to atomically replace 'new' with old.
401
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
402
    On win32, if new exists, it must be moved out of the way first,
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
403
    and then deleted.
1711.7.6 by John Arbash Meinel
Change _win32_rename() so that it raises ENOENT *before* it tries any renaming.
404
    """
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
405
    try:
5186.2.11 by Martin Pool
_win32_rename no longer relies on _wrapped_rename
406
        fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
407
    except OSError as e:
1830.3.15 by John Arbash Meinel
On Mac we get EINVAL when renaming cwd
408
        if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY, errno.EINVAL):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
409
            # If we try to rename a non-existant file onto cwd, we get
410
            # EPERM or EACCES instead of ENOENT, this will raise ENOENT
1830.3.15 by John Arbash Meinel
On Mac we get EINVAL when renaming cwd
411
            # if the old path doesn't exist, sometimes we get EACCES
412
            # On Linux, we seem to get EBUSY, on Mac we get EINVAL
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
413
            os.lstat(old)
414
        raise
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
415
416
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
417
def _mac_getcwd():
6619.3.26 by Martin
Fix fallout from 2to3 getcwdu transformation and other test uses
418
    return unicodedata.normalize('NFC', _getcwd())
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
419
420
6468.6.1 by Ross Lagerwall
Change osutils.rename to extend any exception information given from os.rename.
421
def _rename_wrap_exception(rename_func):
422
    """Adds extra information to any exceptions that come from rename().
423
424
    The exception has an updated message and 'old_filename' and 'new_filename'
425
    attributes.
426
    """
427
428
    def _rename_wrapper(old, new):
429
        try:
430
            rename_func(old, new)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
431
        except OSError as e:
6468.6.1 by Ross Lagerwall
Change osutils.rename to extend any exception information given from os.rename.
432
            detailed_error = OSError(e.errno, e.strerror +
433
                                " [occurred when renaming '%s' to '%s']" %
434
                                (old, new))
435
            detailed_error.old_filename = old
436
            detailed_error.new_filename = new
437
            raise detailed_error
438
439
    return _rename_wrapper
440
6619.3.26 by Martin
Fix fallout from 2to3 getcwdu transformation and other test uses
441
442
if sys.version_info > (3,):
443
    _getcwd = os.getcwd
444
else:
445
    _getcwd = os.getcwdu
446
447
6468.6.1 by Ross Lagerwall
Change osutils.rename to extend any exception information given from os.rename.
448
# Default rename wraps os.rename()
449
rename = _rename_wrap_exception(os.rename)
450
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
451
# Default is to just use the python builtins, but these can be rebound on
452
# particular platforms.
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
453
abspath = _posix_abspath
454
realpath = _posix_realpath
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
455
pathjoin = os.path.join
6015.39.2 by Florian Vichot
Fixed an infinite loop when creating a repo at the root of the filesystem,
456
normpath = _posix_normpath
6362.2.1 by Martin Packman
Add path_from_environ function for getting unicode paths from envvars
457
path_from_environ = _posix_path_from_environ
6437.26.1 by Martin Packman
Add and test osutils._get_home_dir for unicode access to home location across platforms
458
_get_home_dir = _posix_get_home_dir
6362.2.4 by Martin Packman
Use native functions for getting username avoiding bug 660174 entirely
459
getuser_unicode = _posix_getuser_unicode
6619.3.26 by Martin
Fix fallout from 2to3 getcwdu transformation and other test uses
460
getcwd = _getcwd
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
461
dirname = os.path.dirname
462
basename = os.path.basename
2215.4.2 by Alexander Belchenko
split and splitext now the part of osutils
463
split = os.path.split
464
splitext = os.path.splitext
5273.1.10 by Vincent Ladeuil
Fixed as per jam's review.
465
# These were already lazily imported into local scope
466
# mkdtemp = tempfile.mkdtemp
467
# rmtree = shutil.rmtree
5609.29.5 by John Arbash Meinel
Fix bug #740932. Transform should update the sha cache.
468
lstat = os.lstat
469
fstat = os.fstat
470
471
def wrap_stat(st):
472
    return st
473
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
474
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
475
MIN_ABS_PATHLENGTH = 1
476
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
477
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
478
if sys.platform == 'win32':
6658.6.1 by Martin
Remove winver, win98 support code, and deprecated code
479
    abspath = _win32_abspath
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
480
    realpath = _win32_realpath
481
    pathjoin = _win32_pathjoin
482
    normpath = _win32_normpath
483
    getcwd = _win32_getcwd
484
    mkdtemp = _win32_mkdtemp
6468.6.1 by Ross Lagerwall
Change osutils.rename to extend any exception information given from os.rename.
485
    rename = _rename_wrap_exception(_win32_rename)
5609.29.5 by John Arbash Meinel
Fix bug #740932. Transform should update the sha cache.
486
    try:
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
487
        from . import _walkdirs_win32
5609.29.5 by John Arbash Meinel
Fix bug #740932. Transform should update the sha cache.
488
    except ImportError:
489
        pass
490
    else:
491
        lstat = _walkdirs_win32.lstat
492
        fstat = _walkdirs_win32.fstat
493
        wrap_stat = _walkdirs_win32.wrap_stat
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
494
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
495
    MIN_ABS_PATHLENGTH = 3
1532 by Robert Collins
Merge in John Meinels integration branch.
496
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
497
    def _win32_delete_readonly(function, path, excinfo):
498
        """Error handler for shutil.rmtree function [for win32]
499
        Helps to remove files and dirs marked as read-only.
500
        """
2116.5.1 by Henri Wiechers
Fixes osutils.rmtree on Windows with Python 2.5
501
        exception = excinfo[1]
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
502
        if function in (os.remove, os.rmdir) \
2116.5.1 by Henri Wiechers
Fixes osutils.rmtree on Windows with Python 2.5
503
            and isinstance(exception, OSError) \
504
            and exception.errno == errno.EACCES:
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
505
            make_writable(path)
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
506
            function(path)
507
        else:
508
            raise
509
510
    def rmtree(path, ignore_errors=False, onerror=_win32_delete_readonly):
511
        """Replacer for shutil.rmtree: could remove readonly dirs/files"""
512
        return shutil.rmtree(path, ignore_errors, onerror)
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
513
514
    f = win32utils.get_unicode_argv     # special function or None
515
    if f is not None:
516
        get_unicode_argv = f
6362.2.1 by Martin Packman
Add path_from_environ function for getting unicode paths from envvars
517
    path_from_environ = win32utils.get_environ_unicode
6437.26.1 by Martin Packman
Add and test osutils._get_home_dir for unicode access to home location across platforms
518
    _get_home_dir = win32utils.get_home_location
6362.2.4 by Martin Packman
Use native functions for getting username avoiding bug 660174 entirely
519
    getuser_unicode = win32utils.get_user_name
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
520
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
521
elif sys.platform == 'darwin':
522
    getcwd = _mac_getcwd
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
523
1685.1.31 by John Arbash Meinel
Adding tests for the rest of the _win32 functions.
524
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
525
def get_terminal_encoding(trace=False):
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
526
    """Find the best encoding for printing to the screen.
527
528
    This attempts to check both sys.stdout and sys.stdin to see
529
    what encoding they are in, and if that fails it falls back to
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
530
    osutils.get_user_encoding().
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
531
    The problem is that on Windows, locale.getpreferredencoding()
532
    is not the same encoding as that used by the console:
533
    http://mail.python.org/pipermail/python-list/2003-May/162357.html
534
535
    On my standard US Windows XP, the preferred encoding is
536
    cp1252, but the console is cp437
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
537
538
    :param trace: If True trace the selected encoding via mutter().
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
539
    """
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
540
    from .trace import mutter
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
541
    output_encoding = getattr(sys.stdout, 'encoding', None)
542
    if not output_encoding:
543
        input_encoding = getattr(sys.stdin, 'encoding', None)
544
        if not input_encoding:
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
545
            output_encoding = get_user_encoding()
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
546
            if trace:
547
                mutter('encoding stdout as osutils.get_user_encoding() %r',
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
548
                   output_encoding)
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
549
        else:
550
            output_encoding = input_encoding
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
551
            if trace:
552
                mutter('encoding stdout as sys.stdin encoding %r',
553
                    output_encoding)
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
554
    else:
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
555
        if trace:
556
            mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
2127.4.1 by Alexander Belchenko
(jam, bialix) Workaround for cp0 console encoding on Windows
557
    if output_encoding == 'cp0':
558
        # invalid encoding (cp0 means 'no codepage' on Windows)
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
559
        output_encoding = get_user_encoding()
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
560
        if trace:
561
            mutter('cp0 is invalid encoding.'
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
562
               ' encoding stdout as osutils.get_user_encoding() %r',
563
               output_encoding)
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
564
    # check encoding
565
    try:
566
        codecs.lookup(output_encoding)
567
    except LookupError:
6622.1.4 by Jelmer Vernooij
Fix some more tests.
568
        sys.stderr.write('brz: warning:'
2192.1.9 by Alexander Belchenko
final fix suggested by John Meinel
569
                         ' unknown terminal encoding %s.\n'
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
570
                         '  Using encoding %s instead.\n'
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
571
                         % (output_encoding, get_user_encoding())
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
572
                        )
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
573
        output_encoding = get_user_encoding()
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
574
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
575
    return output_encoding
576
577
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
578
def normalizepath(f):
3287.18.2 by Matt McClure
Reverts to 3290.
579
    if getattr(os.path, 'realpath', None) is not None:
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
580
        F = realpath
581
    else:
582
        F = abspath
583
    [p,e] = os.path.split(f)
584
    if e == "" or e == "." or e == "..":
585
        return F(f)
586
    else:
587
        return pathjoin(F(p), e)
588
1 by mbp at sourcefrog
import from baz patch-364
589
590
def isdir(f):
591
    """True if f is an accessible directory."""
592
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
593
        return stat.S_ISDIR(os.lstat(f)[stat.ST_MODE])
1 by mbp at sourcefrog
import from baz patch-364
594
    except OSError:
595
        return False
596
597
598
def isfile(f):
599
    """True if f is a regular file."""
600
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
601
        return stat.S_ISREG(os.lstat(f)[stat.ST_MODE])
1 by mbp at sourcefrog
import from baz patch-364
602
    except OSError:
603
        return False
604
1092.2.6 by Robert Collins
symlink support updated to work
605
def islink(f):
606
    """True if f is a symlink."""
607
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
608
        return stat.S_ISLNK(os.lstat(f)[stat.ST_MODE])
1092.2.6 by Robert Collins
symlink support updated to work
609
    except OSError:
610
        return False
1 by mbp at sourcefrog
import from baz patch-364
611
485 by Martin Pool
- move commit code into its own module
612
def is_inside(dir, fname):
613
    """True if fname is inside dir.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
614
1185.31.38 by John Arbash Meinel
Changing os.path.normpath to osutils.normpath
615
    The parameters should typically be passed to osutils.normpath first, so
969 by Martin Pool
- Add less-sucky is_within_any
616
    that . and .. and repeated slashes are eliminated, and the separators
617
    are canonical for the platform.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
618
619
    The empty string as a dir name is taken as top-of-tree and matches
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
620
    everything.
485 by Martin Pool
- move commit code into its own module
621
    """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
622
    # XXX: Most callers of this can actually do something smarter by
969 by Martin Pool
- Add less-sucky is_within_any
623
    # looking at the inventory
972 by Martin Pool
- less dodgy is_inside function
624
    if dir == fname:
625
        return True
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
626
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
627
    if dir == '':
628
        return True
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
629
1185.31.34 by John Arbash Meinel
Removing instances of os.sep
630
    if dir[-1] != '/':
631
        dir += '/'
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
632
972 by Martin Pool
- less dodgy is_inside function
633
    return fname.startswith(dir)
634
485 by Martin Pool
- move commit code into its own module
635
636
def is_inside_any(dir_list, fname):
637
    """True if fname is inside any of given dirs."""
638
    for dirname in dir_list:
639
        if is_inside(dirname, fname):
640
            return True
2324.2.3 by Dmitry Vasiliev
Fixed is_inside_* methods implementation
641
    return False
485 by Martin Pool
- move commit code into its own module
642
643
1740.3.4 by Jelmer Vernooij
Move inventory to commit builder.
644
def is_inside_or_parent_of_any(dir_list, fname):
645
    """True if fname is a child or a parent of any of the given files."""
646
    for dirname in dir_list:
647
        if is_inside(dirname, fname) or is_inside(fname, dirname):
648
            return True
2324.2.3 by Dmitry Vasiliev
Fixed is_inside_* methods implementation
649
    return False
1740.3.4 by Jelmer Vernooij
Move inventory to commit builder.
650
651
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
652
def pumpfile(from_file, to_file, read_length=-1, buff_size=32768,
653
             report_activity=None, direction='read'):
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
654
    """Copy contents of one file to another.
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
655
656
    The read_length can either be -1 to read to end-of-file (EOF) or
657
    it can specify the maximum number of bytes to read.
658
659
    The buff_size represents the maximum size for each read operation
660
    performed on from_file.
661
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
662
    :param report_activity: Call this as bytes are read, see
663
        Transport._report_activity
664
    :param direction: Will be passed to report_activity
665
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
666
    :return: The number of bytes copied.
667
    """
668
    length = 0
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
669
    if read_length >= 0:
670
        # read specified number of bytes
671
672
        while read_length > 0:
673
            num_bytes_to_read = min(read_length, buff_size)
674
675
            block = from_file.read(num_bytes_to_read)
676
            if not block:
677
                # EOF reached
678
                break
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
679
            if report_activity is not None:
680
                report_activity(len(block), direction)
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
681
            to_file.write(block)
682
683
            actual_bytes_read = len(block)
684
            read_length -= actual_bytes_read
685
            length += actual_bytes_read
686
    else:
687
        # read to EOF
688
        while True:
689
            block = from_file.read(buff_size)
690
            if not block:
691
                # EOF reached
692
                break
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
693
            if report_activity is not None:
694
                report_activity(len(block), direction)
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
695
            to_file.write(block)
696
            length += len(block)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
697
    return length
1 by mbp at sourcefrog
import from baz patch-364
698
699
3635.1.2 by Robert Collins
Add osutils.pump_string_file helper function.
700
def pump_string_file(bytes, file_handle, segment_size=None):
701
    """Write bytes to file_handle in many smaller writes.
702
703
    :param bytes: The string to write.
704
    :param file_handle: The file to write to.
705
    """
706
    # Write data in chunks rather than all at once, because very large
707
    # writes fail on some platforms (e.g. Windows with SMB  mounted
708
    # drives).
709
    if not segment_size:
710
        segment_size = 5242880 # 5MB
6632.1.1 by Martin
Change uses of buffer to memoryview
711
    offsets = range(0, len(bytes), segment_size)
712
    view = memoryview(bytes)
3635.1.2 by Robert Collins
Add osutils.pump_string_file helper function.
713
    write = file_handle.write
6632.1.1 by Martin
Change uses of buffer to memoryview
714
    for offset in offsets:
715
        write(view[offset:offset+segment_size])
3635.1.2 by Robert Collins
Add osutils.pump_string_file helper function.
716
717
1185.67.7 by Aaron Bentley
Refactored a bit
718
def file_iterator(input_file, readsize=32768):
719
    while True:
720
        b = input_file.read(readsize)
721
        if len(b) == 0:
722
            break
723
        yield b
724
725
1 by mbp at sourcefrog
import from baz patch-364
726
def sha_file(f):
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
727
    """Calculate the hexdigest of an open file.
728
729
    The file cursor should be already at the start.
730
    """
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
731
    s = sha()
320 by Martin Pool
- Compute SHA-1 of files in chunks
732
    BUFSIZE = 128<<10
733
    while True:
734
        b = f.read(BUFSIZE)
735
        if not b:
736
            break
737
        s.update(b)
1 by mbp at sourcefrog
import from baz patch-364
738
    return s.hexdigest()
739
740
3368.2.49 by Ian Clatworthy
added osutils.size_sha_file() with tests
741
def size_sha_file(f):
742
    """Calculate the size and hexdigest of an open file.
743
744
    The file cursor should be already at the start and
745
    the caller is responsible for closing the file afterwards.
746
    """
747
    size = 0
748
    s = sha()
749
    BUFSIZE = 128<<10
750
    while True:
751
        b = f.read(BUFSIZE)
752
        if not b:
753
            break
754
        size += len(b)
755
        s.update(b)
756
    return size, s.hexdigest()
757
758
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
759
def sha_file_by_name(fname):
760
    """Calculate the SHA1 of a file by reading the full text"""
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
761
    s = sha()
4634.140.1 by INADA Naoki
Avoids child process inherits file handles on win32. by using os.fdopen and os.open with O_NOINHERIT instead of builtin open.
762
    f = os.open(fname, os.O_RDONLY | O_BINARY | O_NOINHERIT)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
763
    try:
2872.3.2 by Martin Pool
Do sha_file_by_name using raw os files rather than file objects; makes this routine about 12osutils.py faster
764
        while True:
765
            b = os.read(f, 1<<16)
766
            if not b:
767
                return s.hexdigest()
768
            s.update(b)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
769
    finally:
2872.3.2 by Martin Pool
Do sha_file_by_name using raw os files rather than file objects; makes this routine about 12osutils.py faster
770
        os.close(f)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
771
772
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
773
def sha_strings(strings, _factory=sha):
1235 by Martin Pool
- split sha_strings into osutils
774
    """Return the sha-1 of concatenation of strings"""
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
775
    s = _factory()
6631.3.1 by Martin
Run 2to3 map fixer and refactor after
776
    for string in strings:
777
        s.update(string)
1235 by Martin Pool
- split sha_strings into osutils
778
    return s.hexdigest()
779
780
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
781
def sha_string(f, _factory=sha):
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
782
    return _factory(f).hexdigest()
1 by mbp at sourcefrog
import from baz patch-364
783
784
124 by mbp at sourcefrog
- check file text for past revisions is correct
785
def fingerprint_file(f):
126 by mbp at sourcefrog
Use just one big read to fingerprint files
786
    b = f.read()
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
787
    return {'size': len(b),
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
788
            'sha1': sha(b).hexdigest()}
124 by mbp at sourcefrog
- check file text for past revisions is correct
789
790
1 by mbp at sourcefrog
import from baz patch-364
791
def compare_files(a, b):
792
    """Returns true if equal in contents"""
74 by mbp at sourcefrog
compare_files: read in one page at a time rather than
793
    BUFSIZE = 4096
794
    while True:
795
        ai = a.read(BUFSIZE)
796
        bi = b.read(BUFSIZE)
797
        if ai != bi:
798
            return False
799
        if ai == '':
800
            return True
1 by mbp at sourcefrog
import from baz patch-364
801
802
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
803
def gmtime(seconds=None):
804
    """Convert seconds since the Epoch to a time tuple expressing UTC (a.k.a.
805
    GMT). When 'seconds' is not passed in, convert the current time instead.
806
    Handy replacement for time.gmtime() buggy on Windows and 32-bit platforms.
807
    """
808
    if seconds is None:
809
        seconds = time.time()
810
    return (datetime(1970, 1, 1) + timedelta(seconds=seconds)).timetuple()
811
812
49 by mbp at sourcefrog
fix local-time-offset calculation
813
def local_time_offset(t=None):
814
    """Return offset of local zone from GMT, either at present or at time t."""
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
815
    if t is None:
73 by mbp at sourcefrog
fix time.localtime call for python 2.3
816
        t = time.time()
2215.6.1 by James Henstridge
Don't rely on time.timezone and time.altzone in local_time_offset(),
817
    offset = datetime.fromtimestamp(t) - datetime.utcfromtimestamp(t)
818
    return offset.days * 86400 + offset.seconds
8 by mbp at sourcefrog
store committer's timezone in revision and show
819
3512.3.1 by Martin von Gagern
Hand-selected minimalistic set of changes from my setlocale branch.
820
weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
4379.4.1 by Ian Clatworthy
make log --long faster
821
_default_format_by_weekday_num = [wd + " %Y-%m-%d %H:%M:%S" for wd in weekdays]
822
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
823
2425.6.2 by Martin Pool
Make timestamps use existing format_date; document that function more
824
def format_date(t, offset=0, timezone='original', date_fmt=None,
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
825
                show_offset=True):
2425.6.2 by Martin Pool
Make timestamps use existing format_date; document that function more
826
    """Return a formatted date string.
827
828
    :param t: Seconds since the epoch.
829
    :param offset: Timezone offset in seconds east of utc.
830
    :param timezone: How to display the time: 'utc', 'original' for the
831
         timezone specified by offset, or 'local' for the process's current
832
         timezone.
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
833
    :param date_fmt: strftime format.
834
    :param show_offset: Whether to append the timezone.
835
    """
836
    (date_fmt, tt, offset_str) = \
837
               _format_date(t, offset, timezone, date_fmt, show_offset)
838
    date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
839
    date_str = time.strftime(date_fmt, tt)
840
    return date_str + offset_str
841
4379.4.1 by Ian Clatworthy
make log --long faster
842
843
# Cache of formatted offset strings
844
_offset_cache = {}
845
846
4379.4.2 by Ian Clatworthy
add NEWS item and tests for new date formatting API
847
def format_date_with_offset_in_original_timezone(t, offset=0,
4379.4.1 by Ian Clatworthy
make log --long faster
848
    _cache=_offset_cache):
849
    """Return a formatted date string in the original timezone.
850
851
    This routine may be faster then format_date.
852
853
    :param t: Seconds since the epoch.
854
    :param offset: Timezone offset in seconds east of utc.
855
    """
856
    if offset is None:
857
        offset = 0
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
858
    tt = gmtime(t + offset)
4379.4.1 by Ian Clatworthy
make log --long faster
859
    date_fmt = _default_format_by_weekday_num[tt[6]]
860
    date_str = time.strftime(date_fmt, tt)
861
    offset_str = _cache.get(offset, None)
862
    if offset_str is None:
863
        offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
864
        _cache[offset] = offset_str
865
    return date_str + offset_str
866
867
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
868
def format_local_date(t, offset=0, timezone='original', date_fmt=None,
869
                      show_offset=True):
870
    """Return an unicode date string formatted according to the current locale.
871
872
    :param t: Seconds since the epoch.
873
    :param offset: Timezone offset in seconds east of utc.
874
    :param timezone: How to display the time: 'utc', 'original' for the
875
         timezone specified by offset, or 'local' for the process's current
876
         timezone.
877
    :param date_fmt: strftime format.
878
    :param show_offset: Whether to append the timezone.
879
    """
880
    (date_fmt, tt, offset_str) = \
881
               _format_date(t, offset, timezone, date_fmt, show_offset)
882
    date_str = time.strftime(date_fmt, tt)
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
883
    if not isinstance(date_str, text_type):
4385.4.1 by Alexander Belchenko
removed all references to bzrlib.user_encoding
884
        date_str = date_str.decode(get_user_encoding(), 'replace')
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
885
    return date_str + offset_str
886
4379.4.1 by Ian Clatworthy
make log --long faster
887
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
888
def _format_date(t, offset, timezone, date_fmt, show_offset):
8 by mbp at sourcefrog
store committer's timezone in revision and show
889
    if timezone == 'utc':
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
890
        tt = gmtime(t)
1 by mbp at sourcefrog
import from baz patch-364
891
        offset = 0
8 by mbp at sourcefrog
store committer's timezone in revision and show
892
    elif timezone == 'original':
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
893
        if offset is None:
23 by mbp at sourcefrog
format_date: handle revisions with no timezone offset
894
            offset = 0
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
895
        tt = gmtime(t + offset)
12 by mbp at sourcefrog
new --timezone option for bzr log
896
    elif timezone == 'local':
1 by mbp at sourcefrog
import from baz patch-364
897
        tt = time.localtime(t)
49 by mbp at sourcefrog
fix local-time-offset calculation
898
        offset = local_time_offset(t)
12 by mbp at sourcefrog
new --timezone option for bzr log
899
    else:
3144.1.1 by Lukáš Lalinský
Fixed error reporting of unsupported timezone format.
900
        raise errors.UnsupportedTimezoneFormat(timezone)
1185.12.24 by Aaron Bentley
Made format_date more flexible
901
    if date_fmt is None:
902
        date_fmt = "%a %Y-%m-%d %H:%M:%S"
903
    if show_offset:
904
        offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
905
    else:
906
        offset_str = ''
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
907
    return (date_fmt, tt, offset_str)
1 by mbp at sourcefrog
import from baz patch-364
908
909
910
def compact_date(when):
6621.1.1 by Florent Gallaire
Fix for Windows and 32-bit platforms buggy gmtime().
911
    return time.strftime('%Y%m%d%H%M%S', gmtime(when))
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
912
1 by mbp at sourcefrog
import from baz patch-364
913
1957.1.4 by John Arbash Meinel
create a helper for formatting a time delta
914
def format_delta(delta):
915
    """Get a nice looking string for a time delta.
916
917
    :param delta: The time difference in seconds, can be positive or negative.
918
        positive indicates time in the past, negative indicates time in the
919
        future. (usually time.time() - stored_time)
920
    :return: String formatted to show approximate resolution
921
    """
922
    delta = int(delta)
923
    if delta >= 0:
924
        direction = 'ago'
925
    else:
926
        direction = 'in the future'
927
        delta = -delta
928
929
    seconds = delta
930
    if seconds < 90: # print seconds up to 90 seconds
931
        if seconds == 1:
932
            return '%d second %s' % (seconds, direction,)
933
        else:
934
            return '%d seconds %s' % (seconds, direction)
935
936
    minutes = int(seconds / 60)
937
    seconds -= 60 * minutes
938
    if seconds == 1:
939
        plural_seconds = ''
940
    else:
941
        plural_seconds = 's'
942
    if minutes < 90: # print minutes, seconds up to 90 minutes
943
        if minutes == 1:
944
            return '%d minute, %d second%s %s' % (
945
                    minutes, seconds, plural_seconds, direction)
946
        else:
947
            return '%d minutes, %d second%s %s' % (
948
                    minutes, seconds, plural_seconds, direction)
949
950
    hours = int(minutes / 60)
951
    minutes -= 60 * hours
952
    if minutes == 1:
953
        plural_minutes = ''
954
    else:
955
        plural_minutes = 's'
956
957
    if hours == 1:
958
        return '%d hour, %d minute%s %s' % (hours, minutes,
959
                                            plural_minutes, direction)
960
    return '%d hours, %d minute%s %s' % (hours, minutes,
961
                                         plural_minutes, direction)
1 by mbp at sourcefrog
import from baz patch-364
962
963
def filesize(f):
964
    """Return size of given open file."""
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
965
    return os.fstat(f.fileno())[stat.ST_SIZE]
1 by mbp at sourcefrog
import from baz patch-364
966
1553.5.5 by Martin Pool
New utility routine rand_chars
967
6419.1.1 by Martin Packman
Simplify urandom alias osutils.rand_bytes based on present realities
968
# Alias os.urandom to support platforms (which?) without /dev/urandom and 
969
# override if it doesn't work. Avoid checking on windows where there is
970
# significant initialisation cost that can be avoided for some bzr calls.
971
972
rand_bytes = os.urandom
973
974
if rand_bytes.__module__ != "nt":
2067.1.1 by John Arbash Meinel
Catch an exception while opening /dev/urandom rather than using os.path.exists()
975
    try:
6419.1.1 by Martin Packman
Simplify urandom alias osutils.rand_bytes based on present realities
976
        rand_bytes(1)
977
    except NotImplementedError:
1185.1.7 by Robert Collins
Nathaniel McCallums patch for urandom friendliness on aix.
978
        # not well seeded, but better than nothing
979
        def rand_bytes(n):
980
            import random
981
            s = ''
982
            while n:
983
                s += chr(random.randint(0, 255))
984
                n -= 1
985
            return s
1 by mbp at sourcefrog
import from baz patch-364
986
1553.5.5 by Martin Pool
New utility routine rand_chars
987
988
ALNUM = '0123456789abcdefghijklmnopqrstuvwxyz'
989
def rand_chars(num):
990
    """Return a random string of num alphanumeric characters
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
991
992
    The result only contains lowercase chars because it may be used on
1553.5.5 by Martin Pool
New utility routine rand_chars
993
    case-insensitive filesystems.
994
    """
995
    s = ''
996
    for raw_byte in rand_bytes(num):
997
        s += ALNUM[ord(raw_byte) % 36]
998
    return s
999
1000
1 by mbp at sourcefrog
import from baz patch-364
1001
## TODO: We could later have path objects that remember their list
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
1002
## decomposition (might be too tricksy though.)
1 by mbp at sourcefrog
import from baz patch-364
1003
1004
def splitpath(p):
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1005
    """Turn string into list of parts."""
271 by Martin Pool
- Windows path fixes
1006
    # split on either delimiter because people might use either on
1007
    # Windows
1008
    ps = re.split(r'[\\/]', p)
1009
1010
    rps = []
1 by mbp at sourcefrog
import from baz patch-364
1011
    for f in ps:
1012
        if f == '..':
6138.3.8 by Jonathan Riddell
more error gettext()ing
1013
            raise errors.BzrError(gettext("sorry, %r not allowed in path") % f)
271 by Martin Pool
- Windows path fixes
1014
        elif (f == '.') or (f == ''):
1015
            pass
1016
        else:
1017
            rps.append(f)
1018
    return rps
1 by mbp at sourcefrog
import from baz patch-364
1019
3890.2.4 by John Arbash Meinel
Add a new function that can convert 'chunks' format to a 'lines' format.
1020
1 by mbp at sourcefrog
import from baz patch-364
1021
def joinpath(p):
1022
    for f in p:
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
1023
        if (f == '..') or (f is None) or (f == ''):
6138.3.8 by Jonathan Riddell
more error gettext()ing
1024
            raise errors.BzrError(gettext("sorry, %r not allowed in path") % f)
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
1025
    return pathjoin(*p)
1 by mbp at sourcefrog
import from baz patch-364
1026
1027
4370.1.1 by Ian Clatworthy
add osutils.parent_directories() API
1028
def parent_directories(filename):
4371.1.1 by Ian Clatworthy
(igc) added osutils.parent_directories() (Ian Clatworthy)
1029
    """Return the list of parent directories, deepest first.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1030
4371.1.1 by Ian Clatworthy
(igc) added osutils.parent_directories() (Ian Clatworthy)
1031
    For example, parent_directories("a/b/c") -> ["a/b", "a"].
1032
    """
4370.1.1 by Ian Clatworthy
add osutils.parent_directories() API
1033
    parents = []
1034
    parts = splitpath(dirname(filename))
1035
    while parts:
1036
        parents.append(joinpath(parts))
1037
        parts.pop()
1038
    return parents
1039
1040
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1041
_extension_load_failures = []
1042
1043
1044
def failed_to_load_extension(exception):
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1045
    """Handle failing to load a binary extension.
1046
1047
    This should be called from the ImportError block guarding the attempt to
1048
    import the native extension.  If this function returns, the pure-Python
1049
    implementation should be loaded instead::
1050
1051
    >>> try:
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
1052
    >>>     import breezy._fictional_extension_pyx
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1053
    >>> except ImportError, e:
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
1054
    >>>     breezy.osutils.failed_to_load_extension(e)
1055
    >>>     import breezy._fictional_extension_py
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1056
    """
1057
    # NB: This docstring is just an example, not a doctest, because doctest
1058
    # currently can't cope with the use of lazy imports in this namespace --
1059
    # mbp 20090729
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1060
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1061
    # This currently doesn't report the failure at the time it occurs, because
1062
    # they tend to happen very early in startup when we can't check config
1063
    # files etc, and also we want to report all failures but not spam the user
1064
    # with 10 warnings.
1065
    exception_str = str(exception)
1066
    if exception_str not in _extension_load_failures:
5523.2.3 by Parth Malwankar
message is now shown to the user but is not too scary.
1067
        trace.mutter("failed to load compiled extension: %s" % exception_str)
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1068
        _extension_load_failures.append(exception_str)
1069
1070
1071
def report_extension_load_failures():
1072
    if not _extension_load_failures:
1073
        return
6059.4.2 by Vincent Ladeuil
Migrate ignore_missing_extensions to stack-based config.
1074
    if config.GlobalStack().get('ignore_missing_extensions'):
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1075
        return
1076
    # the warnings framework should by default show this only once
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
1077
    from .trace import warning
4695.4.1 by Martin Pool
Give a shorter/cleaner message for missing extensions
1078
    warning(
6622.1.4 by Jelmer Vernooij
Fix some more tests.
1079
        "brz: warning: some compiled extensions could not be loaded; "
4695.4.1 by Martin Pool
Give a shorter/cleaner message for missing extensions
1080
        "see <https://answers.launchpad.net/bzr/+faq/703>")
1081
    # we no longer show the specific missing extensions here, because it makes
1082
    # the message too long and scary - see
1083
    # https://bugs.launchpad.net/bzr/+bug/430529
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1084
1085
3890.2.7 by John Arbash Meinel
A Pyrex extension is about 5x faster than the fastest python code I could write.
1086
try:
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
1087
    from ._chunks_to_lines_pyx import chunks_to_lines
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1088
except ImportError as e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1089
    failed_to_load_extension(e)
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
1090
    from ._chunks_to_lines_py import chunks_to_lines
3890.2.7 by John Arbash Meinel
A Pyrex extension is about 5x faster than the fastest python code I could write.
1091
1092
1231 by Martin Pool
- more progress on fetch on top of weaves
1093
def split_lines(s):
1094
    """Split s into lines, but without removing the newline characters."""
3890.2.18 by John Arbash Meinel
Implement osutils.split_lines() in terms of chunks_to_lines if possible.
1095
    # Trivially convert a fulltext into a 'chunked' representation, and let
1096
    # chunks_to_lines do the heavy lifting.
1097
    if isinstance(s, str):
1098
        # chunks_to_lines only supports 8-bit strings
1099
        return chunks_to_lines([s])
1100
    else:
1101
        return _split_lines(s)
1102
1103
1104
def _split_lines(s):
1105
    """Split s into lines, but without removing the newline characters.
1106
1107
    This supports Unicode or plain string objects.
1108
    """
1666.1.6 by Robert Collins
Make knit the default format.
1109
    lines = s.split('\n')
1110
    result = [line + '\n' for line in lines[:-1]]
1111
    if lines[-1]:
1112
        result.append(lines[-1])
1113
    return result
1391 by Robert Collins
merge from integration
1114
1115
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1116
def hardlinks_good():
1185.10.5 by Aaron Bentley
Fixed hardlinks_good test
1117
    return sys.platform not in ('win32', 'cygwin', 'darwin')
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1118
1185.1.46 by Robert Collins
Aarons branch --basis patch
1119
1185.10.3 by Aaron Bentley
Made copy_multi_immutable create hardlinks opportunistically
1120
def link_or_copy(src, dest):
1121
    """Hardlink a file, or copy it if it can't be hardlinked."""
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1122
    if not hardlinks_good():
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1123
        shutil.copyfile(src, dest)
1185.10.3 by Aaron Bentley
Made copy_multi_immutable create hardlinks opportunistically
1124
        return
1125
    try:
1126
        os.link(src, dest)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1127
    except (OSError, IOError) as e:
1185.10.3 by Aaron Bentley
Made copy_multi_immutable create hardlinks opportunistically
1128
        if e.errno != errno.EXDEV:
1129
            raise
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1130
        shutil.copyfile(src, dest)
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1131
2831.5.2 by Vincent Ladeuil
Review feedback.
1132
1133
def delete_any(path):
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1134
    """Delete a file, symlink or directory.
1135
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1136
    Will delete even if readonly.
1137
    """
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1138
    try:
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1139
       _delete_file_or_dir(path)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1140
    except (OSError, IOError) as e:
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1141
        if e.errno in (errno.EPERM, errno.EACCES):
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1142
            # make writable and try again
1143
            try:
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1144
                make_writable(path)
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1145
            except (OSError, IOError):
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1146
                pass
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1147
            _delete_file_or_dir(path)
1148
        else:
1149
            raise
1150
1151
1152
def _delete_file_or_dir(path):
1153
    # Look Before You Leap (LBYL) is appropriate here instead of Easier to Ask for
1154
    # Forgiveness than Permission (EAFP) because:
1155
    # - root can damage a solaris file system by using unlink,
1156
    # - unlink raises different exceptions on different OSes (linux: EISDIR, win32:
1157
    #   EACCES, OSX: EPERM) when invoked on a directory.
2831.5.2 by Vincent Ladeuil
Review feedback.
1158
    if isdir(path): # Takes care of symlinks
1159
        os.rmdir(path)
1160
    else:
1161
        os.unlink(path)
1558.12.9 by Aaron Bentley
Handle resolving conflicts with directories properly
1162
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1163
1164
def has_symlinks():
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
1165
    if getattr(os, 'symlink', None) is not None:
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1166
        return True
1167
    else:
1168
        return False
2831.5.2 by Vincent Ladeuil
Review feedback.
1169
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1170
3136.1.1 by Aaron Bentley
Add support for hardlinks to TreeTransform
1171
def has_hardlinks():
1172
    if getattr(os, 'link', None) is not None:
1173
        return True
1174
    else:
1175
        return False
1176
1177
3287.18.14 by Matt McClure
Extracted a host_os_dereferences_symlinks method.
1178
def host_os_dereferences_symlinks():
1179
    return (has_symlinks()
3287.18.19 by Matt McClure
Changed tested sys.platform value from 'windows' (mistaken) to 'win32'
1180
            and sys.platform not in ('cygwin', 'win32'))
3287.18.14 by Matt McClure
Extracted a host_os_dereferences_symlinks method.
1181
1182
4241.14.14 by Vincent Ladeuil
Test and implements osutils.readlink().
1183
def readlink(abspath):
1184
    """Return a string representing the path to which the symbolic link points.
1185
1186
    :param abspath: The link absolute unicode path.
1187
1188
    This his guaranteed to return the symbolic link in unicode in all python
1189
    versions.
1190
    """
1191
    link = abspath.encode(_fs_enc)
1192
    target = os.readlink(link)
1193
    target = target.decode(_fs_enc)
1194
    return target
1195
1196
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1197
def contains_whitespace(s):
1198
    """True if there are any whitespace characters in s."""
2249.2.1 by John Arbash Meinel
(John Arbash Meinel) hard-code the whitespace chars to avoid problems in some locales.
1199
    # string.whitespace can include '\xa0' in certain locales, because it is
1200
    # considered "non-breaking-space" as part of ISO-8859-1. But it
1201
    # 1) Isn't a breaking whitespace
1202
    # 2) Isn't one of ' \t\r\n' which are characters we sometimes use as
1203
    #    separators
1204
    # 3) '\xa0' isn't unicode safe since it is >128.
2249.5.16 by John Arbash Meinel
[merge] bzr.dev 2283
1205
1206
    # This should *not* be a unicode set of characters in case the source
1207
    # string is not a Unicode string. We can auto-up-cast the characters since
1208
    # they are ascii, but we don't want to auto-up-cast the string in case it
1209
    # is utf-8
1210
    for ch in ' \t\n\r\v\f':
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1211
        if ch in s:
1212
            return True
1213
    else:
1214
        return False
1215
1216
1217
def contains_linebreaks(s):
1218
    """True if there is any vertical whitespace in s."""
1219
    for ch in '\f\n\r':
1220
        if ch in s:
1221
            return True
1222
    else:
1223
        return False
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1224
1225
1226
def relpath(base, path):
5193.2.1 by Alexander Belchenko
update docstring for osutils.relpath() function.
1227
    """Return path relative to base, or raise PathNotChild exception.
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1228
1229
    The path may be either an absolute path or a path relative to the
1230
    current working directory.
1231
1232
    os.path.commonprefix (python2.4) has a bad bug that it works just
1233
    on string prefixes, assuming that '/u' is a prefix of '/u2'.  This
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
1234
    avoids that problem.
5193.2.1 by Alexander Belchenko
update docstring for osutils.relpath() function.
1235
5193.2.2 by Alexander Belchenko
update wording based on spiv's review.
1236
    NOTE: `base` should not have a trailing slash otherwise you'll get
1237
    PathNotChild exceptions regardless of `path`.
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
1238
    """
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1239
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1240
    if len(base) < MIN_ABS_PATHLENGTH:
1241
        # must have space for e.g. a drive letter
6138.3.8 by Jonathan Riddell
more error gettext()ing
1242
        raise ValueError(gettext('%r is too short to calculate a relative path')
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1243
            % (base,))
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
1244
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1245
    rp = abspath(path)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1246
1247
    s = []
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1248
    head = rp
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1249
    while True:
1250
        if len(head) <= len(base) and head != base:
1251
            raise errors.PathNotChild(rp, base)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1252
        if head == base:
1253
            break
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1254
        head, tail = split(head)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1255
        if tail:
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1256
            s.append(tail)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1257
1185.31.35 by John Arbash Meinel
Couple small fixes, all tests pass on cygwin.
1258
    if s:
4555.2.3 by John Arbash Meinel
Fix a trivial bug that should have been caught earlier. :)
1259
        return pathjoin(*reversed(s))
1185.31.35 by John Arbash Meinel
Couple small fixes, all tests pass on cygwin.
1260
    else:
1261
        return ''
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1262
1263
3794.5.29 by Mark Hammond
cicp_canonical_relpath -> _cicp_canonical_relpath
1264
def _cicp_canonical_relpath(base, path):
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1265
    """Return the canonical path relative to base.
1266
1267
    Like relpath, but on case-insensitive-case-preserving file-systems, this
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1268
    will return the relpath as stored on the file-system rather than in the
1269
    case specified in the input string, for all existing portions of the path.
1270
3794.5.28 by Mark Hammond
Update comments.
1271
    This will cause O(N) behaviour if called for every path in a tree; if you
1272
    have a number of paths to convert, you should use canonical_relpaths().
3794.5.31 by Mark Hammond
bulk of the simple review comments from igc.
1273
    """
1274
    # TODO: it should be possible to optimize this for Windows by using the
1275
    # win32 API FindFiles function to look for the specified name - but using
1276
    # os.listdir() still gives us the correct, platform agnostic semantics in
1277
    # the short term.
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1278
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1279
    rel = relpath(base, path)
1280
    # '.' will have been turned into ''
1281
    if not rel:
1282
        return rel
1283
1284
    abs_base = abspath(base)
1285
    current = abs_base
1286
    _listdir = os.listdir
1287
1288
    # use an explicit iterator so we can easily consume the rest on early exit.
3794.5.36 by Mark Hammond
test for, and fix problem with canonical_relpath when the tail does not exist.
1289
    bit_iter = iter(rel.split('/'))
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1290
    for bit in bit_iter:
1291
        lbit = bit.lower()
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1292
        try:
1293
            next_entries = _listdir(current)
4634.70.3 by John Arbash Meinel
Clean up some terminology, catch a double _listdir request, thanks spiv.
1294
        except OSError: # enoent, eperm, etc
1295
            # We can't find this in the filesystem, so just append the
1296
            # remaining bits.
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1297
            current = pathjoin(current, bit, *list(bit_iter))
1298
            break
4634.70.3 by John Arbash Meinel
Clean up some terminology, catch a double _listdir request, thanks spiv.
1299
        for look in next_entries:
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1300
            if lbit == look.lower():
1301
                current = pathjoin(current, look)
1302
                break
1303
        else:
1304
            # got to the end, nothing matched, so we just return the
1305
            # non-existing bits as they were specified (the filename may be
1306
            # the target of a move, for example).
1307
            current = pathjoin(current, bit, *list(bit_iter))
1308
            break
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1309
    return current[len(abs_base):].lstrip('/')
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1310
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1311
# XXX - TODO - we need better detection/integration of case-insensitive
4241.9.5 by Vincent Ladeuil
Fix unicode related OSX failures.
1312
# file-systems; Linux often sees FAT32 devices (or NFS-mounted OSX
1313
# filesystems), for example, so could probably benefit from the same basic
1314
# support there.  For now though, only Windows and OSX get that support, and
1315
# they get it for *all* file-systems!
4241.9.2 by Vincent Ladeuil
Fix most of cicp related failures on OSX.
1316
if sys.platform in ('win32', 'darwin'):
3794.5.29 by Mark Hammond
cicp_canonical_relpath -> _cicp_canonical_relpath
1317
    canonical_relpath = _cicp_canonical_relpath
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1318
else:
1319
    canonical_relpath = relpath
1320
3794.5.15 by Mark Hammond
Add canonical_relpaths() as a placeholder for a future caching implementation.
1321
def canonical_relpaths(base, paths):
1322
    """Create an iterable to canonicalize a sequence of relative paths.
1323
1324
    The intent is for this implementation to use a cache, vastly speeding
1325
    up multiple transformations in the same directory.
1326
    """
1327
    # but for now, we haven't optimized...
1328
    return [canonical_relpath(base, p) for p in paths]
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1329
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1330
1331
def decode_filename(filename):
1332
    """Decode the filename using the filesystem encoding
1333
1334
    If it is unicode, it is returned.
1335
    Otherwise it is decoded from the the filesystem's encoding. If decoding
1336
    fails, a errors.BadFilenameEncoding exception is raised.
1337
    """
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
1338
    if isinstance(filename, text_type):
5279.2.4 by Eric Moritz
Added the filename_decode
1339
        return filename
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1340
    try:
1341
        return filename.decode(_fs_enc)
1342
    except UnicodeDecodeError:
1343
        raise errors.BadFilenameEncoding(filename, _fs_enc)
1344
5279.2.4 by Eric Moritz
Added the filename_decode
1345
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1346
def safe_unicode(unicode_or_utf8_string):
1347
    """Coerce unicode_or_utf8_string into unicode.
1348
1349
    If it is unicode, it is returned.
4204.2.1 by Matt Nordhoff
Fix a broken sentence in osutils.safe_unicode's docstring
1350
    Otherwise it is decoded from utf-8. If decoding fails, the exception is
1351
    wrapped in a BzrBadParameterNotUnicode exception.
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1352
    """
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
1353
    if isinstance(unicode_or_utf8_string, text_type):
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1354
        return unicode_or_utf8_string
1355
    try:
1356
        return unicode_or_utf8_string.decode('utf8')
1357
    except UnicodeDecodeError:
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
1358
        raise errors.BzrBadParameterNotUnicode(unicode_or_utf8_string)
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1359
1360
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1361
def safe_utf8(unicode_or_utf8_string):
1362
    """Coerce unicode_or_utf8_string to a utf8 string.
1363
1364
    If it is a str, it is returned.
1365
    If it is Unicode, it is encoded into a utf-8 string.
1366
    """
1367
    if isinstance(unicode_or_utf8_string, str):
1368
        # TODO: jam 20070209 This is overkill, and probably has an impact on
1369
        #       performance if we are dealing with lots of apis that want a
1370
        #       utf-8 revision id
1371
        try:
1372
            # Make sure it is a valid utf-8 string
1373
            unicode_or_utf8_string.decode('utf-8')
1374
        except UnicodeDecodeError:
1375
            raise errors.BzrBadParameterNotUnicode(unicode_or_utf8_string)
1376
        return unicode_or_utf8_string
1377
    return unicode_or_utf8_string.encode('utf-8')
1378
1379
6630.1.1 by Jelmer Vernooij
Remove deprecated functionality.
1380
def safe_revision_id(unicode_or_utf8_string):
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1381
    """Revision ids should now be utf8, but at one point they were unicode.
1382
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1383
    :param unicode_or_utf8_string: A possibly Unicode revision_id. (can also be
1384
        utf8 or None).
1385
    :return: None or a utf8 revision id.
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1386
    """
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1387
    if (unicode_or_utf8_string is None
1388
        or unicode_or_utf8_string.__class__ == str):
1389
        return unicode_or_utf8_string
6630.1.1 by Jelmer Vernooij
Remove deprecated functionality.
1390
    raise TypeError('Unicode revision ids are no longer supported. '
1391
                    'Revision id generators should be creating utf8 revision '
1392
                    'ids.')
1393
1394
1395
def safe_file_id(unicode_or_utf8_string):
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1396
    """File ids should now be utf8, but at one point they were unicode.
1397
1398
    This is the same as safe_utf8, except it uses the cached encode functions
1399
    to save a little bit of performance.
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1400
1401
    :param unicode_or_utf8_string: A possibly Unicode file_id. (can also be
1402
        utf8 or None).
1403
    :return: None or a utf8 file id.
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1404
    """
1405
    if (unicode_or_utf8_string is None
1406
        or unicode_or_utf8_string.__class__ == str):
1407
        return unicode_or_utf8_string
6630.1.1 by Jelmer Vernooij
Remove deprecated functionality.
1408
    raise TypeError('Unicode file ids are no longer supported. '
1409
                    'File id generators should be creating utf8 file ids.')
2294.1.4 by John Arbash Meinel
Add safe_file_id as a helper in osutils.
1410
1411
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1412
_platform_normalizes_filenames = False
1413
if sys.platform == 'darwin':
1414
    _platform_normalizes_filenames = True
1415
1416
1417
def normalizes_filenames():
1418
    """Return True if this platform normalizes unicode filenames.
1419
5283.2.1 by Martin Pool
Additional platform name tweaks
1420
    Only Mac OSX.
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1421
    """
1422
    return _platform_normalizes_filenames
1423
1424
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1425
def _accessible_normalized_filename(path):
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1426
    """Get the unicode normalized path, and if you can access the file.
1427
1428
    On platforms where the system normalizes filenames (Mac OSX),
1429
    you can access a file by any path which will normalize correctly.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1430
    On platforms where the system does not normalize filenames
5283.2.1 by Martin Pool
Additional platform name tweaks
1431
    (everything else), you have to access a file by its exact path.
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1432
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1433
    Internally, bzr only supports NFC normalization, since that is
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1434
    the standard for XML documents.
1435
1436
    So return the normalized path, and a flag indicating if the file
1437
    can be accessed by that path.
1438
    """
1439
3201.1.1 by jameinel
Fix bug #185458, switch from NFKC to NFC and add tests for filenames that would be broken under NFKC
1440
    return unicodedata.normalize('NFC', unicode(path)), True
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1441
1442
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1443
def _inaccessible_normalized_filename(path):
1444
    __doc__ = _accessible_normalized_filename.__doc__
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1445
3201.1.1 by jameinel
Fix bug #185458, switch from NFKC to NFC and add tests for filenames that would be broken under NFKC
1446
    normalized = unicodedata.normalize('NFC', unicode(path))
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1447
    return normalized, normalized == path
1448
1449
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1450
if _platform_normalizes_filenames:
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1451
    normalized_filename = _accessible_normalized_filename
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1452
else:
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1453
    normalized_filename = _inaccessible_normalized_filename
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1454
1455
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1456
def set_signal_handler(signum, handler, restart_syscall=True):
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1457
    """A wrapper for signal.signal that also calls siginterrupt(signum, False)
1458
    on platforms that support that.
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1459
1460
    :param restart_syscall: if set, allow syscalls interrupted by a signal to
1461
        automatically restart (by calling `signal.siginterrupt(signum,
1462
        False)`).  May be ignored if the feature is not available on this
1463
        platform or Python version.
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1464
    """
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1465
    try:
5169.2.2 by Vincent Ladeuil
Just rely on python to tell us what it supports.
1466
        import signal
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1467
        siginterrupt = signal.siginterrupt
5169.2.2 by Vincent Ladeuil
Just rely on python to tell us what it supports.
1468
    except ImportError:
1469
        # This python implementation doesn't provide signal support, hence no
1470
        # handler exists
1471
        return None
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1472
    except AttributeError:
1473
        # siginterrupt doesn't exist on this platform, or for this version
1474
        # of Python.
1475
        siginterrupt = lambda signum, flag: None
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1476
    if restart_syscall:
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1477
        def sig_handler(*args):
1478
            # Python resets the siginterrupt flag when a signal is
5141.4.3 by Andrew Bennetts
Link to Python bug in comment.
1479
            # received.  <http://bugs.python.org/issue8354>
1480
            # As a workaround for some cases, set it back the way we want it.
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1481
            siginterrupt(signum, False)
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1482
            # Now run the handler function passed to set_signal_handler.
1483
            handler(*args)
1484
    else:
1485
        sig_handler = handler
1486
    old_handler = signal.signal(signum, sig_handler)
1487
    if restart_syscall:
1488
        siginterrupt(signum, False)
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1489
    return old_handler
1490
1491
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1492
default_terminal_width = 80
1493
"""The default terminal width for ttys.
1494
1495
This is defined so that higher levels can share a common fallback value when
1496
terminal_width() returns None.
1497
"""
1498
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1499
# Keep some state so that terminal_width can detect if _terminal_size has
1500
# returned a different size since the process started.  See docstring and
1501
# comments of terminal_width for details.
1502
# _terminal_size_state has 3 possible values: no_data, unchanged, and changed.
1503
_terminal_size_state = 'no_data'
1504
_first_terminal_size = None
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1505
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1506
def terminal_width():
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1507
    """Return terminal width.
1508
1509
    None is returned if the width can't established precisely.
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1510
1511
    The rules are:
6622.1.28 by Jelmer Vernooij
More renames; commands in output, environment variables.
1512
    - if BRZ_COLUMNS is set, returns its value
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1513
    - if there is no controlling terminal, returns None
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1514
    - query the OS, if the queried size has changed since the last query,
1515
      return its value,
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1516
    - if COLUMNS is set, returns its value,
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1517
    - if the OS has a value (even though it's never changed), return its value.
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1518
1519
    From there, we need to query the OS to get the size of the controlling
1520
    terminal.
1521
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1522
    On Unices we query the OS by:
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1523
    - get termios.TIOCGWINSZ
1524
    - if an error occurs or a negative value is obtained, returns None
1525
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1526
    On Windows we query the OS by:
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1527
    - win32utils.get_console_size() decides,
1528
    - returns None on error (provided default value)
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1529
    """
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1530
    # Note to implementors: if changing the rules for determining the width,
1531
    # make sure you've considered the behaviour in these cases:
1532
    #  - M-x shell in emacs, where $COLUMNS is set and TIOCGWINSZ returns 0,0.
6622.1.30 by Jelmer Vernooij
Some more test fixes.
1533
    #  - brz log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1534
    #    0,0.
1535
    #  - (add more interesting cases here, if you find any)
1536
    # Some programs implement "Use $COLUMNS (if set) until SIGWINCH occurs",
1537
    # but we don't want to register a signal handler because it is impossible
1538
    # to do so without risking EINTR errors in Python <= 2.6.5 (see
1539
    # <http://bugs.python.org/issue8354>).  Instead we check TIOCGWINSZ every
1540
    # time so we can notice if the reported size has changed, which should have
1541
    # a similar effect.
4747.3.4 by Vincent Ladeuil
Add tests, introduce explicit default values, always respect COLUMNS.
1542
6622.1.28 by Jelmer Vernooij
More renames; commands in output, environment variables.
1543
    # If BRZ_COLUMNS is set, take it, user is always right
5582.7.1 by Neil Martinsen-Burrell
allow BZR_COLUMNS to be 0
1544
    # Except if they specified 0 in which case, impose no limit here
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1545
    try:
6622.1.28 by Jelmer Vernooij
More renames; commands in output, environment variables.
1546
        width = int(os.environ['BRZ_COLUMNS'])
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1547
    except (KeyError, ValueError):
5582.7.1 by Neil Martinsen-Burrell
allow BZR_COLUMNS to be 0
1548
        width = None
1549
    if width is not None:
1550
        if width > 0:
1551
            return width
1552
        else:
1553
            return None
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1554
4747.3.3 by Vincent Ladeuil
More complete fix (previous one changed the focus).
1555
    isatty = getattr(sys.stdout, 'isatty', None)
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1556
    if isatty is None or not isatty():
6622.1.28 by Jelmer Vernooij
More renames; commands in output, environment variables.
1557
        # Don't guess, setting BRZ_COLUMNS is the recommended way to override.
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1558
        return None
4747.3.1 by Joke de Buhr
Prevent linebreaks in output if it's not connected to a tty.
1559
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1560
    # Query the OS
1561
    width, height = os_size = _terminal_size(None, None)
1562
    global _first_terminal_size, _terminal_size_state
1563
    if _terminal_size_state == 'no_data':
1564
        _first_terminal_size = os_size
1565
        _terminal_size_state = 'unchanged'
1566
    elif (_terminal_size_state == 'unchanged' and
1567
          _first_terminal_size != os_size):
1568
        _terminal_size_state = 'changed'
1569
1570
    # If the OS claims to know how wide the terminal is, and this value has
1571
    # ever changed, use that.
1572
    if _terminal_size_state == 'changed':
1573
        if width is not None and width > 0:
1574
            return width
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1575
1576
    # If COLUMNS is set, use it.
4747.4.3 by Vincent Ladeuil
Re-fix the priority order since there is a known valid case.
1577
    try:
1578
        return int(os.environ['COLUMNS'])
1579
    except (KeyError, ValueError):
1580
        pass
1581
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1582
    # Finally, use an unchanged size from the OS, if we have one.
1583
    if _terminal_size_state == 'unchanged':
1584
        if width is not None and width > 0:
1585
            return width
1586
1587
    # The width could not be determined.
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1588
    return None
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1589
1590
1591
def _win32_terminal_size(width, height):
1592
    width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1593
    return width, height
1594
1595
1596
def _ioctl_terminal_size(width, height):
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1597
    try:
1704.2.2 by Martin Pool
Detect terminal width using ioctl
1598
        import struct, fcntl, termios
1599
        s = struct.pack('HHHH', 0, 0, 0, 0)
1600
        x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
4747.4.6 by Vincent Ladeuil
Fix parameter order.
1601
        height, width = struct.unpack('HHHH', x)[0:2]
4747.3.4 by Vincent Ladeuil
Add tests, introduce explicit default values, always respect COLUMNS.
1602
    except (IOError, AttributeError):
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1603
        pass
1604
    return width, height
1605
1606
_terminal_size = None
1607
"""Returns the terminal size as (width, height).
1608
1609
:param width: Default value for width.
1610
:param height: Default value for height.
1611
1612
This is defined specifically for each OS and query the size of the controlling
1613
terminal. If any error occurs, the provided default values should be returned.
1614
"""
1615
if sys.platform == 'win32':
1616
    _terminal_size = _win32_terminal_size
1617
else:
1618
    _terminal_size = _ioctl_terminal_size
1534.7.25 by Aaron Bentley
Added set_executability
1619
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1620
1534.7.25 by Aaron Bentley
Added set_executability
1621
def supports_executable():
1534.7.160 by Aaron Bentley
Changed implementation of supports_executable
1622
    return sys.platform != "win32"
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
1623
1624
1551.10.4 by Aaron Bentley
Update to skip on win32
1625
def supports_posix_readonly():
1626
    """Return True if 'readonly' has POSIX semantics, False otherwise.
1627
1628
    Notably, a win32 readonly file cannot be deleted, unlike POSIX where the
1629
    directory controls creation/deletion, etc.
1630
1631
    And under win32, readonly means that the directory itself cannot be
1632
    deleted.  The contents of a readonly directory can be changed, unlike POSIX
1633
    where files in readonly directories cannot be added, deleted or renamed.
1634
    """
1635
    return sys.platform != "win32"
1636
1637
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1638
def set_or_unset_env(env_variable, value):
1639
    """Modify the environment, setting or removing the env_variable.
1640
1641
    :param env_variable: The environment variable in question
1642
    :param value: The value to set the environment to. If None, then
1643
        the variable will be removed.
1644
    :return: The original value of the environment variable.
1645
    """
1646
    orig_val = os.environ.get(env_variable)
1647
    if value is None:
1648
        if orig_val is not None:
1649
            del os.environ[env_variable]
1650
    else:
6621.2.26 by Martin
Misc set of changes to get started with selftest on Python 3
1651
        if not PY3 and isinstance(value, text_type):
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
1652
            value = value.encode(get_user_encoding())
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1653
        os.environ[env_variable] = value
1654
    return orig_val
1655
1656
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
1657
_validWin32PathRE = re.compile(r'^([A-Za-z]:[/\\])?[^:<>*"?\|]*$')
1658
1659
1660
def check_legal_path(path):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1661
    """Check whether the supplied path is legal.
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
1662
    This is only required on Windows, so we don't test on other platforms
1663
    right now.
1664
    """
1665
    if sys.platform != "win32":
1666
        return
1667
    if _validWin32PathRE.match(path) is None:
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
1668
        raise errors.IllegalPath(path)
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1669
1670
3596.2.2 by John Arbash Meinel
Factor out the common exception handling looking for ENOTDIR and use it
1671
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1672
1673
def _is_error_enotdir(e):
1674
    """Check if this exception represents ENOTDIR.
1675
1676
    Unfortunately, python is very inconsistent about the exception
1677
    here. The cases are:
1678
      1) Linux, Mac OSX all versions seem to set errno == ENOTDIR
1679
      2) Windows, Python2.4, uses errno == ERROR_DIRECTORY (267)
1680
         which is the windows error code.
1681
      3) Windows, Python2.5 uses errno == EINVAL and
1682
         winerror == ERROR_DIRECTORY
1683
1684
    :param e: An Exception object (expected to be OSError with an errno
1685
        attribute, but we should be able to cope with anything)
1686
    :return: True if this represents an ENOTDIR error. False otherwise.
1687
    """
1688
    en = getattr(e, 'errno', None)
1689
    if (en == errno.ENOTDIR
1690
        or (sys.platform == 'win32'
1691
            and (en == _WIN32_ERROR_DIRECTORY
1692
                 or (en == errno.EINVAL
1693
                     and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1694
        ))):
1695
        return True
1696
    return False
1697
1698
1757.2.8 by Robert Collins
Teach walkdirs to walk a subdir of a tree.
1699
def walkdirs(top, prefix=""):
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1700
    """Yield data about all the directories in a tree.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1701
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1702
    This yields all the data about the contents of a directory at a time.
1703
    After each directory has been yielded, if the caller has mutated the list
1704
    to exclude some directories, they are then not descended into.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1705
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1706
    The data yielded is of the form:
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1707
    ((directory-relpath, directory-path-from-top),
2694.4.1 by Alexander Belchenko
trivial fix for docstring of osutils.walkdirs()
1708
    [(relpath, basename, kind, lstat, path-from-top), ...]),
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1709
     - directory-relpath is the relative path of the directory being returned
1710
       with respect to top. prefix is prepended to this.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1711
     - directory-path-from-root is the path including top for this directory.
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1712
       It is suitable for use with os functions.
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1713
     - relpath is the relative path within the subtree being walked.
1714
     - basename is the basename of the path
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1715
     - kind is the kind of the file now. If unknown then the file is not
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1716
       present within the tree - but it may be recorded as versioned. See
1717
       versioned_kind.
1718
     - lstat is the stat data *if* the file was statted.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1719
     - planned, not implemented:
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1720
       path_from_tree_root is the path from the root of the tree.
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1721
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1722
    :param prefix: Prefix the relpaths that are yielded with 'prefix'. This
1757.2.16 by Robert Collins
Review comments.
1723
        allows one to walk a subtree but get paths that are relative to a tree
1724
        rooted higher up.
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1725
    :return: an iterator over the dirs.
1726
    """
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1727
    #TODO there is a bit of a smell where the results of the directory-
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1728
    # summary in this, and the path from the root, may not agree
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1729
    # depending on top and prefix - i.e. ./foo and foo as a pair leads to
1730
    # potentially confusing output. We should make this more robust - but
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1731
    # not at a speed cost. RBC 20060731
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1732
    _lstat = os.lstat
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1733
    _directory = _directory_kind
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1734
    _listdir = os.listdir
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1735
    _kind_from_mode = file_kind_from_stat_mode
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1736
    pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1737
    while pending:
1738
        # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1739
        relroot, _, _, _, top = pending.pop()
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1740
        if relroot:
1741
            relprefix = relroot + u'/'
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1742
        else:
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1743
            relprefix = ''
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1744
        top_slash = top + u'/'
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1745
1746
        dirblock = []
1747
        append = dirblock.append
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1748
        try:
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1749
            names = sorted(map(decode_filename, _listdir(top)))
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1750
        except OSError as e:
3596.2.2 by John Arbash Meinel
Factor out the common exception handling looking for ENOTDIR and use it
1751
            if not _is_error_enotdir(e):
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1752
                raise
1753
        else:
1754
            for name in names:
1755
                abspath = top_slash + name
1756
                statvalue = _lstat(abspath)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1757
                kind = _kind_from_mode(statvalue.st_mode)
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1758
                append((relprefix + name, name, kind, statvalue, abspath))
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1759
        yield (relroot, top), dirblock
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1760
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1761
        # push the user specified dirs from dirblock
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1762
        pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1763
1764
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1765
class DirReader(object):
1766
    """An interface for reading directories."""
1767
1768
    def top_prefix_to_starting_dir(self, top, prefix=""):
1769
        """Converts top and prefix to a starting dir entry
1770
1771
        :param top: A utf8 path
1772
        :param prefix: An optional utf8 path to prefix output relative paths
1773
            with.
1774
        :return: A tuple starting with prefix, and ending with the native
1775
            encoding of top.
1776
        """
1777
        raise NotImplementedError(self.top_prefix_to_starting_dir)
1778
1779
    def read_dir(self, prefix, top):
1780
        """Read a specific dir.
1781
1782
        :param prefix: A utf8 prefix to be preprended to the path basenames.
1783
        :param top: A natively encoded path to read.
3696.3.10 by Robert Collins
Review feedback.
1784
        :return: A list of the directories contents. Each item contains:
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1785
            (utf8_relpath, utf8_name, kind, lstatvalue, native_abspath)
1786
        """
1787
        raise NotImplementedError(self.read_dir)
1788
1789
1790
_selected_dir_reader = None
1791
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1792
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1793
def _walkdirs_utf8(top, prefix=""):
1794
    """Yield data about all the directories in a tree.
1795
1796
    This yields the same information as walkdirs() only each entry is yielded
1797
    in utf-8. On platforms which have a filesystem encoding of utf8 the paths
1798
    are returned as exact byte-strings.
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1799
1800
    :return: yields a tuple of (dir_info, [file_info])
1801
        dir_info is (utf8_relpath, path-from-top)
1802
        file_info is (utf8_relpath, utf8_name, kind, lstat, path-from-top)
1803
        if top is an absolute path, path-from-top is also an absolute path.
1804
        path-from-top might be unicode or utf8, but it is the correct path to
1805
        pass to os functions to affect the file in question. (such as os.lstat)
1806
    """
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1807
    global _selected_dir_reader
1808
    if _selected_dir_reader is None:
6658.6.1 by Martin
Remove winver, win98 support code, and deprecated code
1809
        if sys.platform == "win32":
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1810
            try:
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
1811
                from ._walkdirs_win32 import Win32ReadDir
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1812
                _selected_dir_reader = Win32ReadDir()
4241.14.6 by Vincent Ladeuil
Start DirReader parametrized tests.
1813
            except ImportError:
1814
                pass
6352.3.4 by Martin Packman
Minor tweaks including normalising _fs_enc value
1815
        elif _fs_enc in ('utf-8', 'ascii'):
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1816
            try:
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
1817
                from ._readdir_pyx import UTF8DirReader
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1818
                _selected_dir_reader = UTF8DirReader()
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1819
            except ImportError as e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1820
                failed_to_load_extension(e)
4241.14.6 by Vincent Ladeuil
Start DirReader parametrized tests.
1821
                pass
1822
1823
    if _selected_dir_reader is None:
1824
        # Fallback to the python version
1825
        _selected_dir_reader = UnicodeDirReader()
1826
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1827
    # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1828
    # But we don't actually uses 1-3 in pending, so set them to None
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1829
    pending = [[_selected_dir_reader.top_prefix_to_starting_dir(top, prefix)]]
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1830
    read_dir = _selected_dir_reader.read_dir
1831
    _directory = _directory_kind
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1832
    while pending:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1833
        relroot, _, _, _, top = pending[-1].pop()
1834
        if not pending[-1]:
1835
            pending.pop()
1836
        dirblock = sorted(read_dir(relroot, top))
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1837
        yield (relroot, top), dirblock
1838
        # push the user specified dirs from dirblock
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1839
        next = [d for d in reversed(dirblock) if d[2] == _directory]
1840
        if next:
1841
            pending.append(next)
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1842
1843
1844
class UnicodeDirReader(DirReader):
1845
    """A dir reader for non-utf8 file systems, which transcodes."""
1846
1847
    __slots__ = ['_utf8_encode']
1848
1849
    def __init__(self):
1850
        self._utf8_encode = codecs.getencoder('utf8')
1851
1852
    def top_prefix_to_starting_dir(self, top, prefix=""):
1853
        """See DirReader.top_prefix_to_starting_dir."""
1854
        return (safe_utf8(prefix), None, None, None, safe_unicode(top))
1855
1856
    def read_dir(self, prefix, top):
1857
        """Read a single directory from a non-utf8 file system.
1858
1859
        top, and the abspath element in the output are unicode, all other paths
1860
        are utf8. Local disk IO is done via unicode calls to listdir etc.
1861
1862
        This is currently the fallback code path when the filesystem encoding is
1863
        not UTF-8. It may be better to implement an alternative so that we can
1864
        safely handle paths that are not properly decodable in the current
1865
        encoding.
1866
1867
        See DirReader.read_dir for details.
1868
        """
1869
        _utf8_encode = self._utf8_encode
1870
        _lstat = os.lstat
1871
        _listdir = os.listdir
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1872
        _kind_from_mode = file_kind_from_stat_mode
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1873
1874
        if prefix:
1875
            relprefix = prefix + '/'
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1876
        else:
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1877
            relprefix = ''
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1878
        top_slash = top + u'/'
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1879
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1880
        dirblock = []
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1881
        append = dirblock.append
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1882
        for name in sorted(_listdir(top)):
3696.3.12 by Robert Collins
Fix PQM test failure.
1883
            try:
1884
                name_utf8 = _utf8_encode(name)[0]
1885
            except UnicodeDecodeError:
1886
                raise errors.BadFilenameEncoding(
1887
                    _utf8_encode(relprefix)[0] + name, _fs_enc)
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1888
            abspath = top_slash + name
1889
            statvalue = _lstat(abspath)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1890
            kind = _kind_from_mode(statvalue.st_mode)
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1891
            append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1892
        return dirblock
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1893
1894
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1895
def copy_tree(from_path, to_path, handlers={}):
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1896
    """Copy all of the entries in from_path into to_path.
1897
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1898
    :param from_path: The base directory to copy.
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1899
    :param to_path: The target directory. If it does not exist, it will
1900
        be created.
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1901
    :param handlers: A dictionary of functions, which takes a source and
1902
        destinations for files, directories, etc.
1903
        It is keyed on the file kind, such as 'directory', 'symlink', or 'file'
1904
        'file', 'directory', and 'symlink' should always exist.
1905
        If they are missing, they will be replaced with 'os.mkdir()',
1906
        'os.readlink() + os.symlink()', and 'shutil.copy2()', respectively.
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1907
    """
1908
    # Now, just copy the existing cached tree to the new location
1909
    # We use a cheap trick here.
1910
    # Absolute paths are prefixed with the first parameter
1911
    # relative paths are prefixed with the second.
1912
    # So we can get both the source and target returned
1913
    # without any extra work.
1914
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1915
    def copy_dir(source, dest):
1916
        os.mkdir(dest)
1917
1918
    def copy_link(source, dest):
1919
        """Copy the contents of a symlink"""
1920
        link_to = os.readlink(source)
1921
        os.symlink(link_to, dest)
1922
1923
    real_handlers = {'file':shutil.copy2,
1924
                     'symlink':copy_link,
1925
                     'directory':copy_dir,
1926
                    }
1927
    real_handlers.update(handlers)
1928
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1929
    if not os.path.exists(to_path):
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1930
        real_handlers['directory'](from_path, to_path)
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1931
1932
    for dir_info, entries in walkdirs(from_path, prefix=to_path):
1933
        for relpath, name, kind, st, abspath in entries:
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1934
            real_handlers[kind](abspath, relpath)
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1935
1936
5116.2.6 by Parth Malwankar
renamed copy_ownership to copy_ownership_from_path.
1937
def copy_ownership_from_path(dst, src=None):
5051.4.11 by Parth Malwankar
closed Martins review comments.
1938
    """Copy usr/grp ownership from src file/dir to dst file/dir.
1939
1940
    If src is None, the containing directory is used as source. If chown
1941
    fails, the error is ignored and a warning is printed.
1942
    """
5074.4.6 by John Arbash Meinel
Unbreak bzr on windows.
1943
    chown = getattr(os, 'chown', None)
1944
    if chown is None:
1945
        return
5051.4.9 by Parth Malwankar
removed parent_dir.
1946
1947
    if src == None:
1948
        src = os.path.dirname(dst)
1949
        if src == '':
1950
            src = '.'
1951
4634.143.4 by Parth Malwankar
added parent_dir and mkdir to osutils. osutils.mkdir optionally
1952
    try:
4634.143.1 by Parth Malwankar
default .bazaar, .bzr.log and .bazaar/bazaar.conf retain
1953
        s = os.stat(src)
5074.4.6 by John Arbash Meinel
Unbreak bzr on windows.
1954
        chown(dst, s.st_uid, s.st_gid)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
1955
    except OSError as e:
5523.2.3 by Parth Malwankar
message is now shown to the user but is not too scary.
1956
        trace.warning(
1957
            'Unable to copy ownership from "%s" to "%s". '
1958
            'You may want to set it manually.', src, dst)
1959
        trace.log_exception_quietly()
4634.143.4 by Parth Malwankar
added parent_dir and mkdir to osutils. osutils.mkdir optionally
1960
1961
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1962
def path_prefix_key(path):
1963
    """Generate a prefix-order path key for path.
1964
1965
    This can be used to sort paths in the same way that walkdirs does.
1966
    """
1773.3.2 by Robert Collins
New corner case from John Meinel, showing up the need to check the directory lexographically outside of a single tree's root. Fixed.
1967
    return (dirname(path) , path)
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1968
1969
1970
def compare_paths_prefix_order(path_a, path_b):
1971
    """Compare path_a and path_b to generate the same order walkdirs uses."""
1972
    key_a = path_prefix_key(path_a)
1973
    key_b = path_prefix_key(path_b)
1974
    return cmp(key_a, key_b)
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1975
1976
1977
_cached_user_encoding = None
1978
1979
6630.1.1 by Jelmer Vernooij
Remove deprecated functionality.
1980
def get_user_encoding():
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1981
    """Find out what the preferred user encoding is.
1982
1983
    This is generally the encoding that is used for command line parameters
1984
    and file contents. This may be different from the terminal encoding
1985
    or the filesystem encoding.
1986
1987
    :return: A string defining the preferred user encoding
1988
    """
1989
    global _cached_user_encoding
6383.1.3 by Martin Packman
Deprecate public use_cache parameter in favour of test specific override
1990
    if _cached_user_encoding is not None:
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1991
        return _cached_user_encoding
1992
6383.1.1 by Martin Packman
Simplify get_user_encoding by avoiding locale hacks and assuming setlocale has been called
1993
    if os.name == 'posix' and getattr(locale, 'CODESET', None) is not None:
1994
        # Use the existing locale settings and call nl_langinfo directly
1995
        # rather than going through getpreferredencoding. This avoids
1996
        # <http://bugs.python.org/issue6202> on OSX Python 2.6 and the
1997
        # possibility of the setlocale call throwing an error.
1998
        user_encoding = locale.nl_langinfo(locale.CODESET)
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1999
    else:
6383.1.1 by Martin Packman
Simplify get_user_encoding by avoiding locale hacks and assuming setlocale has been called
2000
        # GZ 2011-12-19: On windows could call GetACP directly instead.
6383.1.4 by Martin Packman
Simplify tests a little and make it clear setlocale is not being used
2001
        user_encoding = locale.getpreferredencoding(False)
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
2002
6383.1.1 by Martin Packman
Simplify get_user_encoding by avoiding locale hacks and assuming setlocale has been called
2003
    try:
2004
        user_encoding = codecs.lookup(user_encoding).name
2005
    except LookupError:
2006
        if user_encoding not in ("", "cp0"):
6622.1.4 by Jelmer Vernooij
Fix some more tests.
2007
            sys.stderr.write('brz: warning:'
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
2008
                             ' unknown encoding %s.'
2009
                             ' Continuing with ascii encoding.\n'
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
2010
                             % user_encoding
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
2011
                            )
6383.1.1 by Martin Packman
Simplify get_user_encoding by avoiding locale hacks and assuming setlocale has been called
2012
        user_encoding = 'ascii'
2013
    else:
2014
        # Get 'ascii' when setlocale has not been called or LANG=C or unset.
2015
        if user_encoding == 'ascii':
2016
            if sys.platform == 'darwin':
2017
                # OSX is special-cased in Python to have a UTF-8 filesystem
2018
                # encoding and previously had LANG set here if not present.
2019
                user_encoding = 'utf-8'
2020
            # GZ 2011-12-19: Maybe UTF-8 should be the default in this case
2021
            #                for some other posix platforms as well.
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
2022
6383.1.3 by Martin Packman
Deprecate public use_cache parameter in favour of test specific override
2023
    _cached_user_encoding = user_encoding
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
2024
    return user_encoding
2091.1.1 by Martin Pool
Avoid MSG_WAITALL as it doesn't work on Windows
2025
2026
4797.57.10 by Alexander Belchenko
path_encoding selection logic extracted as helper function
2027
def get_diff_header_encoding():
5258.1.5 by Alexander Belchenko
tweak requested by Martin Pool: use terminal encoding for diff headers on all platforms
2028
    return get_terminal_encoding()
4797.57.10 by Alexander Belchenko
path_encoding selection logic extracted as helper function
2029
2030
3626.1.1 by Mark Hammond
Add osutils.get_host_name() to return a unicode hostname to prevent
2031
def get_host_name():
3626.1.4 by John Arbash Meinel
Document the difference in get_host_name, per Robert's request.
2032
    """Return the current unicode host name.
2033
2034
    This is meant to be used in place of socket.gethostname() because that
2035
    behaves inconsistently on different platforms.
2036
    """
3626.1.1 by Mark Hammond
Add osutils.get_host_name() to return a unicode hostname to prevent
2037
    if sys.platform == "win32":
2038
        return win32utils.get_host_name()
2039
    else:
2040
        import socket
2041
        return socket.gethostname().decode(get_user_encoding())
2042
2043
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2044
# We must not read/write any more than 64k at a time from/to a socket so we
2045
# don't risk "no buffer space available" errors on some platforms.  Windows in
2046
# particular is likely to throw WSAECONNABORTED or WSAENOBUFS if given too much
2047
# data at once.
2048
MAX_SOCKET_CHUNK = 64 * 1024
2049
6437.74.2 by John Arbash Meinel
Push the change down into osutils.send_all. Needs tests at that level.
2050
_end_of_stream_errors = [errno.ECONNRESET, errno.EPIPE, errno.EINVAL]
5599.3.1 by John Arbash Meinel
Consider WSAECONNABORTED to be an end-of-stream as well as WSAECONNRESET.
2051
for _eno in ['WSAECONNRESET', 'WSAECONNABORTED']:
2052
    _eno = getattr(errno, _eno, None)
2053
    if _eno is not None:
2054
        _end_of_stream_errors.append(_eno)
2055
del _eno
2056
2057
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2058
def read_bytes_from_socket(sock, report_activity=None,
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2059
        max_read_size=MAX_SOCKET_CHUNK):
2060
    """Read up to max_read_size of bytes from sock and notify of progress.
2061
2062
    Translates "Connection reset by peer" into file-like EOF (return an
2063
    empty string rather than raise an error), and repeats the recv if
2064
    interrupted by a signal.
2065
    """
6619.3.18 by Jelmer Vernooij
Run 2to3 idioms fixer.
2066
    while True:
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2067
        try:
2068
            bytes = sock.recv(max_read_size)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2069
        except socket.error as e:
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2070
            eno = e.args[0]
5599.3.1 by John Arbash Meinel
Consider WSAECONNABORTED to be an end-of-stream as well as WSAECONNRESET.
2071
            if eno in _end_of_stream_errors:
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2072
                # The connection was closed by the other side.  Callers expect
2073
                # an empty string to signal end-of-stream.
2074
                return ""
2075
            elif eno == errno.EINTR:
2076
                # Retry the interrupted recv.
2077
                continue
2078
            raise
2079
        else:
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2080
            if report_activity is not None:
2081
                report_activity(len(bytes), 'read')
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2082
            return bytes
2083
2084
2085
def recv_all(socket, count):
2086
    """Receive an exact number of bytes.
2087
2088
    Regular Socket.recv() may return less than the requested number of bytes,
2089
    depending on what's in the OS buffer.  MSG_WAITALL is not available
2090
    on all platforms, but this should work everywhere.  This will return
2091
    less than the requested amount if the remote end closes.
2092
2093
    This isn't optimized and is intended mostly for use in testing.
2094
    """
2095
    b = ''
2096
    while len(b) < count:
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2097
        new = read_bytes_from_socket(socket, None, count - len(b))
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2098
        if new == '':
2099
            break # eof
2100
        b += new
2101
    return b
2102
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2103
2104
def send_all(sock, bytes, report_activity=None):
2105
    """Send all bytes on a socket.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2106
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2107
    Breaks large blocks in smaller chunks to avoid buffering limitations on
2108
    some platforms, and catches EINTR which may be thrown if the send is
2109
    interrupted by a signal.
2110
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2111
    This is preferred to socket.sendall(), because it avoids portability bugs
2112
    and provides activity reporting.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2113
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2114
    :param report_activity: Call this as bytes are read, see
2115
        Transport._report_activity
2116
    """
2117
    sent_total = 0
2118
    byte_count = len(bytes)
6632.1.1 by Martin
Change uses of buffer to memoryview
2119
    view = memoryview(bytes)
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2120
    while sent_total < byte_count:
2121
        try:
6632.1.1 by Martin
Change uses of buffer to memoryview
2122
            sent = sock.send(view[sent_total:sent_total+MAX_SOCKET_CHUNK])
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2123
        except (socket.error, IOError) as e:
6437.74.2 by John Arbash Meinel
Push the change down into osutils.send_all. Needs tests at that level.
2124
            if e.args[0] in _end_of_stream_errors:
2125
                raise errors.ConnectionReset(
2126
                    "Error trying to write to socket", e)
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2127
            if e.args[0] != errno.EINTR:
2128
                raise
2129
        else:
6437.73.1 by John Arbash Meinel
Fix bug #1047309. Treat a series of no-bytes-sent as a ECONNRESET failure.
2130
            if sent == 0:
6437.73.2 by John Arbash Meinel
simplify the fix. Sending 0 bytes seems to always indicate that we have a closed connection.
2131
                raise errors.ConnectionReset('Sending to %s returned 0 bytes'
2132
                                             % (sock,))
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2133
            sent_total += sent
6437.73.1 by John Arbash Meinel
Fix bug #1047309. Treat a series of no-bytes-sent as a ECONNRESET failure.
2134
            if report_activity is not None:
2135
                report_activity(sent, 'write')
3118.2.1 by Andrew Bennetts
(andrew) Fix #115781 by passing no more than 64k at a time to socket.sendall.
2136
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2137
5247.5.29 by Vincent Ladeuil
Fixed as per jam's review.
2138
def connect_socket(address):
2139
    # Slight variation of the socket.create_connection() function (provided by
2140
    # python-2.6) that can fail if getaddrinfo returns an empty list. We also
2141
    # provide it for previous python versions. Also, we don't use the timeout
2142
    # parameter (provided by the python implementation) so we don't implement
2143
    # it either).
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2144
    err = socket.error('getaddrinfo returns an empty list')
2145
    host, port = address
2146
    for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
2147
        af, socktype, proto, canonname, sa = res
2148
        sock = None
2149
        try:
2150
            sock = socket.socket(af, socktype, proto)
2151
            sock.connect(sa)
2152
            return sock
2153
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2154
        except socket.error as err:
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2155
            # 'err' is now the most recent error
2156
            if sock is not None:
2157
                sock.close()
2158
    raise err
5247.3.7 by Vincent Ladeuil
Provide connect_socket (socket.create_connection) for pythons older than 2.6.
2159
3118.2.1 by Andrew Bennetts
(andrew) Fix #115781 by passing no more than 64k at a time to socket.sendall.
2160
2091.3.7 by Aaron Bentley
Rename real_parent to dereferenced_path
2161
def dereference_path(path):
2162
    """Determine the real path to a file.
2163
2164
    All parent elements are dereferenced.  But the file itself is not
2165
    dereferenced.
2166
    :param path: The original path.  May be absolute or relative.
2167
    :return: the real path *to* the file
2168
    """
2091.3.5 by Aaron Bentley
Move realpath functionality into osutils
2169
    parent, base = os.path.split(path)
2170
    # The pathjoin for '.' is a workaround for Python bug #1213894.
2171
    # (initial path components aren't dereferenced)
2172
    return pathjoin(realpath(pathjoin('.', parent)), base)
2681.3.4 by Lukáš Lalinsky
- Rename 'windows' to 'mapi'
2173
2174
2175
def supports_mapi():
2176
    """Return True if we can use MAPI to launch a mail client."""
2177
    return sys.platform == "win32"
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2178
2179
2180
def resource_string(package, resource_name):
2181
    """Load a resource from a package and return it as a string.
2182
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2183
    Note: Only packages that start with breezy are currently supported.
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2184
2185
    This is designed to be a lightweight implementation of resource
2186
    loading in a way which is API compatible with the same API from
2187
    pkg_resources. See
2188
    http://peak.telecommunity.com/DevCenter/PkgResources#basic-resource-access.
2189
    If and when pkg_resources becomes a standard library, this routine
2190
    can delegate to it.
2191
    """
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2192
    # Check package name is within breezy
2193
    if package == "breezy":
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2194
        resource_relpath = resource_name
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2195
    elif package.startswith("breezy."):
2196
        package = package[len("breezy."):].replace('.', os.sep)
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2197
        resource_relpath = pathjoin(package, resource_name)
2198
    else:
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2199
        raise errors.BzrError('resource package %s not in breezy' % package)
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2200
2201
    # Map the resource to a file and read its contents
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2202
    base = dirname(breezy.__file__)
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2203
    if getattr(sys, 'frozen', None):    # bzr.exe
2204
        base = abspath(pathjoin(base, '..', '..'))
4708.2.2 by Martin
Workingtree changes sitting around since November, more explict closing of files in bzrlib
2205
    f = file(pathjoin(base, resource_relpath), "rU")
2206
    try:
2207
        return f.read()
2208
    finally:
2209
        f.close()
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
2210
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2211
def file_kind_from_stat_mode_thunk(mode):
2212
    global file_kind_from_stat_mode
2213
    if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
2214
        try:
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
2215
            from ._readdir_pyx import UTF8DirReader
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2216
            file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2217
        except ImportError as e:
4694.2.1 by John Arbash Meinel
Fix bug #430645, don't issue a warning when failing to import _readdir_pyx the second time.
2218
            # This is one time where we won't warn that an extension failed to
2219
            # load. The extension is never available on Windows anyway.
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
2220
            from ._readdir_py import (
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
2221
                _kind_from_mode as file_kind_from_stat_mode
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2222
                )
2223
    return file_kind_from_stat_mode(mode)
2224
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2225
6046.2.6 by Shannon Weyrick
Add file_stat, and make file_kind use it. file_stat can potentially cache results.
2226
def file_stat(f, _lstat=os.lstat):
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2227
    try:
6046.2.6 by Shannon Weyrick
Add file_stat, and make file_kind use it. file_stat can potentially cache results.
2228
        # XXX cache?
2229
        return _lstat(f)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2230
    except OSError as e:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2231
        if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
2232
            raise errors.NoSuchFile(f)
2233
        raise
2234
6046.2.6 by Shannon Weyrick
Add file_stat, and make file_kind use it. file_stat can potentially cache results.
2235
def file_kind(f, _lstat=os.lstat):
2236
    stat_value = file_stat(f, _lstat)
2237
    return file_kind_from_stat_mode(stat_value.st_mode)
3923.3.1 by Andrew Bennetts
Quick attempt at adding some EINTR-proofing to smart protocol code.
2238
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2239
def until_no_eintr(f, *a, **kw):
2240
    """Run f(*a, **kw), retrying if an EINTR error occurs.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2241
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2242
    WARNING: you must be certain that it is safe to retry the call repeatedly
2243
    if EINTR does occur.  This is typically only true for low-level operations
2244
    like os.read.  If in any doubt, don't use this.
5011.3.5 by Andrew Bennetts
Expand until_no_eintr's docstring more with some explanation for why it is not a complete solution.
2245
2246
    Keep in mind that this is not a complete solution to EINTR.  There is
2247
    probably code in the Python standard library and other dependencies that
2248
    may encounter EINTR if a signal arrives (and there is signal handler for
6622.1.34 by Jelmer Vernooij
Rename brzlib => breezy.
2249
    that signal).  So this function can reduce the impact for IO that breezy
5011.3.5 by Andrew Bennetts
Expand until_no_eintr's docstring more with some explanation for why it is not a complete solution.
2250
    directly controls, but it is not a complete solution.
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2251
    """
2252
    # Borrowed from Twisted's twisted.python.util.untilConcludes function.
2253
    while True:
2254
        try:
2255
            return f(*a, **kw)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2256
        except (IOError, OSError) as e:
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2257
            if e.errno == errno.EINTR:
2258
                continue
2259
            raise
2260
2261
0.16.79 by Aaron Bentley
Remove dependencies on bzrtools
2262
if sys.platform == "win32":
2263
    def getchar():
6379.1.1 by Jelmer Vernooij
Avoid importing tty and termios if we don't have to.
2264
        import msvcrt
0.16.79 by Aaron Bentley
Remove dependencies on bzrtools
2265
        return msvcrt.getch()
2266
else:
2267
    def getchar():
6379.1.1 by Jelmer Vernooij
Avoid importing tty and termios if we don't have to.
2268
        import tty
2269
        import termios
0.16.79 by Aaron Bentley
Remove dependencies on bzrtools
2270
        fd = sys.stdin.fileno()
2271
        settings = termios.tcgetattr(fd)
2272
        try:
2273
            tty.setraw(fd)
2274
            ch = sys.stdin.read(1)
2275
        finally:
2276
            termios.tcsetattr(fd, termios.TCSADRAIN, settings)
2277
        return ch
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2278
6057.1.1 by Martin Pool
Python can now report a platform of 'linux3' on kernel 3, but it's basically the same
2279
if sys.platform.startswith('linux'):
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2280
    def _local_concurrency():
5677.1.2 by Jelmer Vernooij
Use os.sysconf to get the number of CPUs on Linux. Use sysctl to get the number of CPUs on all *BSDs, not just FreeBSD.
2281
        try:
2282
            return os.sysconf('SC_NPROCESSORS_ONLN')
2283
        except (ValueError, OSError, AttributeError):
2284
            return None
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2285
elif sys.platform == 'darwin':
2286
    def _local_concurrency():
2287
        return subprocess.Popen(['sysctl', '-n', 'hw.availcpu'],
2288
                                stdout=subprocess.PIPE).communicate()[0]
5677.1.2 by Jelmer Vernooij
Use os.sysconf to get the number of CPUs on Linux. Use sysctl to get the number of CPUs on all *BSDs, not just FreeBSD.
2289
elif "bsd" in sys.platform:
4413.1.1 by Matthew Fuller
Catch the number of cores on FreeBSD too.
2290
    def _local_concurrency():
2291
        return subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
2292
                                stdout=subprocess.PIPE).communicate()[0]
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2293
elif sys.platform == 'sunos5':
2294
    def _local_concurrency():
2295
        return subprocess.Popen(['psrinfo', '-p',],
2296
                                stdout=subprocess.PIPE).communicate()[0]
2297
elif sys.platform == "win32":
2298
    def _local_concurrency():
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2299
        # This appears to return the number of cores.
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2300
        return os.environ.get('NUMBER_OF_PROCESSORS')
2301
else:
2302
    def _local_concurrency():
2303
        # Who knows ?
2304
        return None
2305
2306
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2307
_cached_local_concurrency = None
2308
2309
def local_concurrency(use_cache=True):
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2310
    """Return how many processes can be run concurrently.
2311
2312
    Rely on platform specific implementations and default to 1 (one) if
2313
    anything goes wrong.
2314
    """
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2315
    global _cached_local_concurrency
4766.3.4 by Matt Nordhoff
Change the environment variable to a global option.
2316
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2317
    if _cached_local_concurrency is not None and use_cache:
2318
        return _cached_local_concurrency
2319
6622.1.28 by Jelmer Vernooij
More renames; commands in output, environment variables.
2320
    concurrency = os.environ.get('BRZ_CONCURRENCY', None)
4766.3.7 by Vincent Ladeuil
Mix BZR_CONCURRENCY and --concurrency so both are available.
2321
    if concurrency is None:
2322
        try:
5677.1.3 by Jelmer Vernooij
Use try/import rather than checking python version strings.
2323
            import multiprocessing
6241.1.2 by Jelmer Vernooij
Simplify try/finally.
2324
            concurrency = multiprocessing.cpu_count()
2325
        except (ImportError, NotImplementedError):
5677.1.3 by Jelmer Vernooij
Use try/import rather than checking python version strings.
2326
            # multiprocessing is only available on Python >= 2.6
6241.1.2 by Jelmer Vernooij
Simplify try/finally.
2327
            # and multiprocessing.cpu_count() isn't implemented on all
2328
            # platforms
5677.1.3 by Jelmer Vernooij
Use try/import rather than checking python version strings.
2329
            try:
2330
                concurrency = _local_concurrency()
2331
            except (OSError, IOError):
2332
                pass
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2333
    try:
2334
        concurrency = int(concurrency)
2335
    except (TypeError, ValueError):
2336
        concurrency = 1
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2337
    if use_cache:
2338
        _cached_concurrency = concurrency
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2339
    return concurrency
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2340
2341
4794.1.15 by Robert Collins
Review feedback.
2342
class UnicodeOrBytesToBytesWriter(codecs.StreamWriter):
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2343
    """A stream writer that doesn't decode str arguments."""
2344
4794.1.21 by Robert Collins
Python 2.4 doesn't use CodecInfo, so do a type check on the result of codecs.lookup.
2345
    def __init__(self, encode, stream, errors='strict'):
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2346
        codecs.StreamWriter.__init__(self, stream, errors)
4794.1.21 by Robert Collins
Python 2.4 doesn't use CodecInfo, so do a type check on the result of codecs.lookup.
2347
        self.encode = encode
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2348
2349
    def write(self, object):
6619.3.18 by Jelmer Vernooij
Run 2to3 idioms fixer.
2350
        if isinstance(object, str):
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2351
            self.stream.write(object)
2352
        else:
2353
            data, _ = self.encode(object, self.errors)
2354
            self.stream.write(data)
4797.2.27 by Vincent Ladeuil
Merge 2.0 into 2.1 including fix for #524560
2355
4634.140.4 by INADA Naoki
Fix easy miss in previous commit.
2356
if sys.platform == 'win32':
4634.140.10 by INADA Naoki
Change name from osutils.open to osutils.open_file
2357
    def open_file(filename, mode='r', bufsize=-1):
4634.140.13 by Vincent Ladeuil
Fix some typos and add a NEWS entry.
2358
        """This function is used to override the ``open`` builtin.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2359
4634.140.13 by Vincent Ladeuil
Fix some typos and add a NEWS entry.
2360
        But it uses O_NOINHERIT flag so the file handle is not inherited by
2361
        child processes.  Deleting or renaming a closed file opened with this
2362
        function is not blocking child processes.
4634.140.6 by INADA Naoki
Add comment to osutils.open()
2363
        """
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2364
        writing = 'w' in mode
2365
        appending = 'a' in mode
2366
        updating = '+' in mode
2367
        binary = 'b' in mode
2368
4634.140.12 by INADA Naoki
small clean up.
2369
        flags = O_NOINHERIT
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2370
        # see http://msdn.microsoft.com/en-us/library/yeby3zcb%28VS.71%29.aspx
2371
        # for flags for each modes.
2372
        if binary:
4634.140.12 by INADA Naoki
small clean up.
2373
            flags |= O_BINARY
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2374
        else:
4634.140.12 by INADA Naoki
small clean up.
2375
            flags |= O_TEXT
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2376
2377
        if writing:
2378
            if updating:
2379
                flags |= os.O_RDWR
2380
            else:
2381
                flags |= os.O_WRONLY
2382
            flags |= os.O_CREAT | os.O_TRUNC
2383
        elif appending:
2384
            if updating:
2385
                flags |= os.O_RDWR
2386
            else:
2387
                flags |= os.O_WRONLY
2388
            flags |= os.O_CREAT | os.O_APPEND
2389
        else: #reading
2390
            if updating:
2391
                flags |= os.O_RDWR
2392
            else:
2393
                flags |= os.O_RDONLY
2394
2395
        return os.fdopen(os.open(filename, flags), mode, bufsize)
4634.140.2 by INADA Naoki
Add osutils.open() that uses O_NOINHERIT on Win32.
2396
else:
4634.140.10 by INADA Naoki
Change name from osutils.open to osutils.open_file
2397
    open_file = open
5187.2.7 by Parth Malwankar
moved getuser_unicode to osutils.
2398
2399
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2400
def available_backup_name(base, exists):
2401
    """Find a non-existing backup file name.
2402
5409.5.6 by Vincent Ladeuil
Add NEWS entry, tweak osutils.available_backup_name docstring.
2403
    This will *not* create anything, this only return a 'free' entry.  This
2404
    should be used for checking names in a directory below a locked
5409.5.8 by Vincent Ladeuil
Be more explicit about race conditions and LBYL being discouraged
2405
    tree/branch/repo to avoid race conditions. This is LBYL (Look Before You
2406
    Leap) and generally discouraged.
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2407
2408
    :param base: The base name.
5409.5.8 by Vincent Ladeuil
Be more explicit about race conditions and LBYL being discouraged
2409
2410
    :param exists: A callable returning True if the path parameter exists.
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2411
    """
2412
    counter = 1
2413
    name = "%s.~%d~" % (base, counter)
2414
    while exists(name):
2415
        counter += 1
5409.1.7 by Vincent Ladeuil
First orphaning implementation (some tests lacking).
2416
        name = "%s.~%d~" % (base, counter)
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2417
    return name
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2418
2419
5582.6.1 by Max Bowsher
Fix socketpair-based SSH transport leaking socket into other child processes.
2420
def set_fd_cloexec(fd):
2421
    """Set a Unix file descriptor's FD_CLOEXEC flag.  Do nothing if platform
2422
    support for this is not available.
2423
    """
2424
    try:
2425
        import fcntl
2426
        old = fcntl.fcntl(fd, fcntl.F_GETFD)
2427
        fcntl.fcntl(fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
2428
    except (ImportError, AttributeError):
2429
        # Either the fcntl module or specific constants are not present
2430
        pass
5321.1.114 by Gordon Tyler
Merged from bzr.dev.
2431
2432
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2433
def find_executable_on_path(name):
2434
    """Finds an executable on the PATH.
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2435
    
2436
    On Windows, this will try to append each extension in the PATHEXT
2437
    environment variable to the name, if it cannot be found with the name
2438
    as given.
2439
    
2440
    :param name: The base name of the executable.
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2441
    :return: The path to the executable found or None.
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2442
    """
5321.1.105 by Gordon Tyler
Fixed find_executable_on_path to only check PATHEXT on win32.
2443
    if sys.platform == 'win32':
2444
        exts = os.environ.get('PATHEXT', '').split(os.pathsep)
5321.1.106 by Gordon Tyler
Fixed find_executable_on_path to properly test for executable-ness on win32 and not split the PATH for each extension in PATHEXT.
2445
        exts = [ext.lower() for ext in exts]
2446
        base, ext = os.path.splitext(name)
2447
        if ext != '':
2448
            if ext.lower() not in exts:
2449
                return None
2450
            name = base
2451
            exts = [ext]
5321.1.105 by Gordon Tyler
Fixed find_executable_on_path to only check PATHEXT on win32.
2452
    else:
2453
        exts = ['']
6437.44.1 by Gordon Tyler
Backport of fix for bug 939605 to bzr 2.5 series.
2454
    path = os.environ.get('PATH')
2455
    if path is not None:
2456
        path = path.split(os.pathsep)
2457
        for ext in exts:
2458
            for d in path:
2459
                f = os.path.join(d, name) + ext
2460
                if os.access(f, os.X_OK):
2461
                    return f
2462
    if sys.platform == 'win32':
2463
        app_path = win32utils.get_app_path(name)
2464
        if app_path != name:
2465
            return app_path
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2466
    return None
5425.5.2 by Martin
Split pid deadness detection code out into osutils
2467
2468
2469
def _posix_is_local_pid_dead(pid):
5425.5.4 by Martin
Get docstring of _posix_is_local_pid_dead the right way round
2470
    """True if pid doesn't correspond to live process on this machine"""
5425.5.2 by Martin
Split pid deadness detection code out into osutils
2471
    try:
2472
        # Special meaning of unix kill: just check if it's there.
2473
        os.kill(pid, 0)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2474
    except OSError as e:
5425.5.2 by Martin
Split pid deadness detection code out into osutils
2475
        if e.errno == errno.ESRCH:
2476
            # On this machine, and really not found: as sure as we can be
2477
            # that it's dead.
2478
            return True
2479
        elif e.errno == errno.EPERM:
2480
            # exists, though not ours
2481
            return False
2482
        else:
5425.4.20 by Martin Pool
Suppress failures from os.kill when just using it to check for process liveness
2483
            mutter("os.kill(%d, 0) failed: %s" % (pid, e))
2484
            # Don't really know.
2485
            return False
5425.5.2 by Martin
Split pid deadness detection code out into osutils
2486
    else:
2487
        # Exists and our process: not dead.
2488
        return False
2489
5425.5.5 by Martin
Quick implementation of dead process detection on win32
2490
if sys.platform == "win32":
2491
    is_local_pid_dead = win32utils.is_local_pid_dead
2492
else:
2493
    is_local_pid_dead = _posix_is_local_pid_dead
6006.4.8 by Martin Pool
Use fsync when fdatasync is not available
2494
6015.60.4 by John Arbash Meinel
Found it called EOPNOTSUPP on a platform, include that spelling as well.
2495
_maybe_ignored = ['EAGAIN', 'EINTR', 'ENOTSUP', 'EOPNOTSUPP', 'EACCES']
2496
_fdatasync_ignored = [getattr(errno, name) for name in _maybe_ignored
6015.60.3 by John Arbash Meinel
only squelch known errors per vila.
2497
                      if getattr(errno, name, None) is not None]
6006.4.8 by Martin Pool
Use fsync when fdatasync is not available
2498
6015.60.4 by John Arbash Meinel
Found it called EOPNOTSUPP on a platform, include that spelling as well.
2499
6006.4.8 by Martin Pool
Use fsync when fdatasync is not available
2500
def fdatasync(fileno):
2501
    """Flush file contents to disk if possible.
2502
    
2503
    :param fileno: Integer OS file handle.
2504
    :raises TransportNotPossible: If flushing to disk is not possible.
2505
    """
2506
    fn = getattr(os, 'fdatasync', getattr(os, 'fsync', None))
2507
    if fn is not None:
6015.60.2 by John Arbash Meinel
Fix bug #1075108.
2508
        try:
2509
            fn(fileno)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2510
        except IOError as e:
6015.60.2 by John Arbash Meinel
Fix bug #1075108.
2511
            # See bug #1075108, on some platforms fdatasync exists, but can
2512
            # raise ENOTSUP. However, we are calling fdatasync to be helpful
2513
            # and reduce the chance of corruption-on-powerloss situations. It
2514
            # is not a mandatory call, so it is ok to suppress failures.
2515
            trace.mutter("ignoring error calling fdatasync: %s" % (e,))
6015.60.3 by John Arbash Meinel
only squelch known errors per vila.
2516
            if getattr(e, 'errno', None) not in _fdatasync_ignored:
2517
                raise
6336.2.1 by Martin Packman
Add is_environment_error() and switch trace to using it
2518
2519
6015.51.1 by Martin Pool
Tolerate empty limbo and pending-deletion directories
2520
def ensure_empty_directory_exists(path, exception_class):
2521
    """Make sure a local directory exists and is empty.
2522
    
2523
    If it does not exist, it is created.  If it exists and is not empty, an
2524
    instance of exception_class is raised.
2525
    """
2526
    try:
2527
        os.mkdir(path)
6619.3.2 by Jelmer Vernooij
Apply 2to3 except fix.
2528
    except OSError as e:
6015.51.1 by Martin Pool
Tolerate empty limbo and pending-deletion directories
2529
        if e.errno != errno.EEXIST:
2530
            raise
2531
        if os.listdir(path) != []:
2532
            raise exception_class(path)
6344.1.1 by Martin Packman
Merge 2.4 into bzr.dev
2533
2534
6336.2.1 by Martin Packman
Add is_environment_error() and switch trace to using it
2535
def is_environment_error(evalue):
2536
    """True if exception instance is due to a process environment issue
2537
2538
    This includes OSError and IOError, but also other errors that come from
2539
    the operating system or core libraries but are not subclasses of those.
2540
    """
2541
    if isinstance(evalue, (EnvironmentError, select.error)):
2542
        return True
2543
    if sys.platform == "win32" and win32utils._is_pywintypes_error(evalue):
2544
        return True
2545
    return False