/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
1739.2.6 by Robert Collins
Merge bzr.dev
1
# Copyright (C) 2006, 2008 Canonical Ltd
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
17
"""Wrapper for readdir which returns files ordered by inode."""
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
18
19
20
import os
21
import sys
22
3731.1.1 by Robert Collins
* The C extensions now build on python 2.4 (Robert Collins, #271939)
23
#python2.4 support
24
cdef extern from "python-compat.h":
25
    pass
26
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
27
28
cdef extern from 'errno.h':
29
    int ENOENT
1739.2.6 by Robert Collins
Merge bzr.dev
30
    int ENOTDIR
31
    int EAGAIN
3766.1.5 by Martin Pool
add missing pyrex import
32
    int EINTR
3766.1.6 by Martin Pool
We need a 'global' declaration to assign to errno; and fix comments
33
    char *strerror(int errno)
34
    # not necessarily a real variable, but this should be close enough
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
35
    int errno
36
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
37
cdef extern from 'unistd.h':
38
    int chdir(char *path)
39
    char *getcwd(char *, int size)
40
41
cdef extern from 'stdlib.h':
42
    void *malloc(int)
43
    void free(void *)
44
3696.3.10 by Robert Collins
Review feedback.
45
46
cdef extern from 'sys/types.h':
47
    ctypedef long ssize_t
48
    ctypedef unsigned long size_t
49
    ctypedef long time_t
50
    ctypedef unsigned long ino_t
51
    ctypedef unsigned long long off_t
52
53
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
54
cdef extern from 'sys/stat.h':
55
    cdef struct stat:
56
        int st_mode
3696.3.10 by Robert Collins
Review feedback.
57
        off_t st_size
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
58
        int st_dev
3696.3.10 by Robert Collins
Review feedback.
59
        ino_t st_ino
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
60
        int st_mtime
61
        int st_ctime
62
    int lstat(char *path, stat *buf)
63
    int S_ISDIR(int mode)
64
    int S_ISCHR(int mode)
65
    int S_ISBLK(int mode)
66
    int S_ISREG(int mode)
67
    int S_ISFIFO(int mode)
68
    int S_ISLNK(int mode)
69
    int S_ISSOCK(int mode)
70
71
72
cdef extern from 'Python.h':
73
    char * PyString_AS_STRING(object)
74
    ctypedef int Py_ssize_t # Required for older pyrex versions
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
75
    ctypedef struct PyObject:
76
        pass
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
77
    Py_ssize_t PyString_Size(object s)
78
    object PyList_GetItem(object lst, Py_ssize_t index)
79
    void *PyList_GetItem_object_void "PyList_GET_ITEM" (object lst, int index)
80
    int PyList_Append(object lst, object item) except -1
81
    void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index)
82
    int PyTuple_SetItem(void *, Py_ssize_t pos, object item) except -1
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
83
    int PyTuple_SetItem_obj "PyTuple_SetItem" (void *, Py_ssize_t pos, PyObject * item) except -1
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
84
    void Py_INCREF(object o)
85
    void Py_DECREF(object o)
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
86
    void PyString_Concat(PyObject **string, object newpart)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
87
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
88
89
cdef extern from 'dirent.h':
90
    ctypedef struct dirent:
91
        char d_name[256]
3696.3.6 by Robert Collins
Partial review feedback fixups.
92
        ino_t d_ino
3766.1.6 by Martin Pool
We need a 'global' declaration to assign to errno; and fix comments
93
    # the opaque C library DIR type.
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
94
    ctypedef struct DIR
95
    # should be DIR *, pyrex barfs.
1739.2.6 by Robert Collins
Merge bzr.dev
96
    DIR * opendir(char * name)
97
    int closedir(DIR * dir)
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
98
    dirent *readdir(DIR *dir)
99
100
_directory = 'directory'
101
_chardev = 'chardev'
102
_block = 'block'
103
_file = 'file'
104
_fifo = 'fifo'
105
_symlink = 'symlink'
106
_socket = 'socket'
107
_unknown = 'unknown'
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
108
_missing = 'missing'
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
109
110
# add a typedef struct dirent dirent to workaround pyrex
111
cdef extern from 'readdir.h':
112
    pass
113
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
114
115
cdef class _Stat:
116
    """Represent a 'stat' result."""
117
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
118
    cdef stat _st
119
120
    property st_dev:
121
        def __get__(self):
122
            return self._st.st_dev
123
124
    property st_ino:
125
        def __get__(self):
126
            return self._st.st_ino
127
128
    property st_mode:
129
        def __get__(self):
130
            return self._st.st_mode
131
132
    property st_ctime:
133
        def __get__(self):
134
            return self._st.st_ctime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
135
136
    property st_mtime:
137
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
138
            return self._st.st_mtime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
139
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
140
    property st_size:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
141
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
142
            return self._st.st_size
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
143
144
    def __repr__(self):
145
        """Repr is the same as a Stat object.
146
3696.3.6 by Robert Collins
Partial review feedback fixups.
147
        (mode, ino, dev, nlink, uid, gid, size, None(atime), mtime, ctime)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
148
        """
149
        return repr((self.st_mode, 0, 0, 0, 0, 0, self.st_size, None,
150
                     self._mtime, self._ctime))
151
152
153
from bzrlib import osutils
154
155
156
cdef class UTF8DirReader:
157
    """A dir reader for utf8 file systems."""
158
159
    cdef readonly object _safe_utf8
160
    cdef _directory, _chardev, _block, _file, _fifo, _symlink
161
    cdef _socket, _unknown
162
163
    def __init__(self):
164
        self._safe_utf8 = osutils.safe_utf8
165
        self._directory = _directory
166
        self._chardev = _chardev
167
        self._block = _block
168
        self._file = _file
169
        self._fifo = _fifo
170
        self._symlink = _symlink
171
        self._socket = _socket
172
        self._unknown = _unknown
173
174
    def kind_from_mode(self, int mode):
175
        """Get the kind of a path from a mode status."""
176
        return self._kind_from_mode(mode)
177
178
    cdef _kind_from_mode(self, int mode):
3696.3.10 by Robert Collins
Review feedback.
179
        # Files and directories are the most common - check them first.
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
180
        if S_ISREG(mode):
181
            return self._file
182
        if S_ISDIR(mode):
183
            return self._directory
184
        if S_ISCHR(mode):
185
            return self._chardev
186
        if S_ISBLK(mode):
187
            return self._block
188
        if S_ISLNK(mode):
189
            return self._symlink
190
        if S_ISFIFO(mode):
191
            return self._fifo
192
        if S_ISSOCK(mode):
193
            return self._socket
194
        return self._unknown
195
196
    def top_prefix_to_starting_dir(self, top, prefix=""):
197
        """See DirReader.top_prefix_to_starting_dir."""
198
        return (self._safe_utf8(prefix), None, None, None,
199
            self._safe_utf8(top))
200
201
    def read_dir(self, prefix, top):
202
        """Read a single directory from a utf8 file system.
203
204
        All paths in and out are utf8.
205
206
        This sub-function is called when we know the filesystem is already in utf8
207
        encoding. So we don't need to transcode filenames.
208
209
        See DirReader.read_dir for details.
210
        """
211
        #cdef char *_prefix = prefix
212
        #cdef char *_top = top
213
        # Use C accelerated directory listing.
214
        cdef object newval
215
        cdef int index
216
        cdef int length
217
        cdef void * atuple
218
        cdef object name
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
219
        cdef PyObject * new_val_obj
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
220
221
        if PyString_Size(prefix):
222
            relprefix = prefix + '/'
223
        else:
224
            relprefix = ''
225
        top_slash = top + '/'
226
227
        # read_dir supplies in should-stat order.
228
        # for _, name in sorted(_listdir(top)):
229
        result = _read_dir(top)
230
        length = len(result)
231
        # result.sort()
232
        for index from 0 <= index < length:
233
            atuple = PyList_GetItem_object_void(result, index)
234
            name = <object>PyTuple_GetItem_void_void(atuple, 1)
3696.3.10 by Robert Collins
Review feedback.
235
            # We have a tuple with (inode, name, None, statvalue, None)
236
            # Now edit it:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
237
            # inode -> path_from_top
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
238
            # direct concat - faster than operator +.
239
            new_val_obj = <PyObject *>relprefix
240
            Py_INCREF(relprefix)
241
            PyString_Concat(&new_val_obj, name)
242
            if NULL == new_val_obj:
243
                # PyString_Concat will have setup an exception, but how to get
244
                # at it?
245
                raise Exception("failed to strcat")
246
            PyTuple_SetItem_obj(atuple, 0, new_val_obj)
3696.3.10 by Robert Collins
Review feedback.
247
            # 1st None -> kind
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
248
            newval = self._kind_from_mode(
249
                (<_Stat>PyTuple_GetItem_void_void(atuple, 3)).st_mode)
250
            Py_INCREF(newval)
251
            PyTuple_SetItem(atuple, 2, newval)
3696.3.10 by Robert Collins
Review feedback.
252
            # 2nd None -> abspath # for all - the caller may need to stat files
253
            # etc.
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
254
            # direct concat - faster than operator +.
255
            new_val_obj = <PyObject *>top_slash
256
            Py_INCREF(top_slash)
257
            PyString_Concat(&new_val_obj, name)
258
            if NULL == new_val_obj:
259
                # PyString_Concat will have setup an exception, but how to get
260
                # at it?
261
                raise Exception("failed to strcat")
262
            PyTuple_SetItem_obj(atuple, 4, new_val_obj)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
263
        return result
264
265
266
cdef _read_dir(path):
1739.2.11 by Robert Collins
Docstring and copyright header update per Martin's review.
267
    """Like os.listdir, this reads the contents of a directory.
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
268
269
    :param path: the directory to list.
3696.3.10 by Robert Collins
Review feedback.
270
    :return: a list of single-owner (the list) tuples ready for editing into
271
        the result tuples walkdirs needs to yield. They contain (inode, name,
272
        None, statvalue, None).
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
273
    """
274
    cdef DIR *the_dir
275
    # currently this needs a fixup - the C code says 'dirent' but should say
276
    # 'struct dirent'
277
    cdef dirent * entry
1739.2.6 by Robert Collins
Merge bzr.dev
278
    cdef dirent sentinel
279
    cdef char *name
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
280
    cdef int stat_result
281
    cdef _Stat statvalue
282
    cdef char *cwd
3766.1.6 by Martin Pool
We need a 'global' declaration to assign to errno; and fix comments
283
    global errno
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
284
285
    cwd = getcwd(NULL, 0)
286
    if -1 == chdir(path):
287
        raise OSError(errno, strerror(errno))
288
    the_dir = opendir(".")
1739.2.6 by Robert Collins
Merge bzr.dev
289
    if NULL == the_dir:
290
        raise OSError(errno, strerror(errno))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
291
    result = []
292
    try:
1739.2.6 by Robert Collins
Merge bzr.dev
293
        entry = &sentinel
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
294
        while entry != NULL:
3766.1.1 by Martin Pool
Fix errors in calling readdir from pyrex (lp:279381)
295
            # Unlike most libc functions, readdir needs errno set to 0
296
            # beforehand so that eof can be distinguished from errors.  See
297
            # <https://bugs.launchpad.net/bzr/+bug/279381>
298
            while True:
299
                errno = 0;
300
                entry = readdir(the_dir)
3766.1.2 by Martin Pool
Be a bit more careful about checking readdir result
301
                if entry == NULL and (errno == EAGAIN or errno == EINTR):
3766.1.1 by Martin Pool
Fix errors in calling readdir from pyrex (lp:279381)
302
                    # try again
303
                    continue
304
                else:
305
                    break
1739.2.6 by Robert Collins
Merge bzr.dev
306
            if entry == NULL:
3766.2.1 by Martin Pool
Don't treat ENOENT from readdir as indicating end-of-directory
307
                if errno == ENOTDIR or errno == 0:
1739.2.6 by Robert Collins
Merge bzr.dev
308
                    # We see ENOTDIR at the end of a normal directory.
309
                    # As ENOTDIR for read_dir(file) is triggered on opendir,
310
                    # we consider ENOTDIR to be 'no error'.
3766.1.1 by Martin Pool
Fix errors in calling readdir from pyrex (lp:279381)
311
                    continue
312
                else:
1739.2.6 by Robert Collins
Merge bzr.dev
313
                    raise OSError(errno, strerror(errno))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
314
            name = entry.d_name
3696.3.6 by Robert Collins
Partial review feedback fixups.
315
            if not (name[0] == c"." and (
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
316
                (name[1] == 0) or 
3696.3.6 by Robert Collins
Partial review feedback fixups.
317
                (name[1] == c"." and name[2] == 0))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
318
                ):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
319
                statvalue = _Stat()
320
                stat_result = lstat(entry.d_name, &statvalue._st)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
321
                if stat_result != 0:
322
                    if errno != ENOENT:
323
                        raise OSError(errno, strerror(errno))
324
                    else:
325
                        kind = _missing
326
                        statvalue = None
327
                # We append a 5-tuple that can be modified in-place by the C
328
                # api:
3696.3.6 by Robert Collins
Partial review feedback fixups.
329
                # inode to sort on (to replace with top_path)
330
                # name (to keep)
331
                # kind (None, to set)
332
                # statvalue (to keep)
333
                # abspath (None, to set)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
334
                PyList_Append(result, (entry.d_ino, entry.d_name, None,
335
                    statvalue, None))
1739.2.6 by Robert Collins
Merge bzr.dev
336
    finally:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
337
        if -1 == chdir(cwd):
338
            free(cwd)
339
            raise OSError(errno, strerror(errno))
340
        free(cwd)
1739.2.6 by Robert Collins
Merge bzr.dev
341
        if -1 == closedir(the_dir):
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
342
            raise OSError(errno, strerror(errno))
343
    return result
1739.2.6 by Robert Collins
Merge bzr.dev
344
345
346
# vim: tw=79 ai expandtab sw=4 sts=4