/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
1739.2.6 by Robert Collins
Merge bzr.dev
1
# Copyright (C) 2006, 2008 Canonical Ltd
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
17
"""Wrapper for readdir which returns files ordered by inode."""
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
18
19
20
import os
21
import sys
22
23
24
# the opaque C library DIR type.
25
cdef extern from 'errno.h':
26
    int ENOENT
1739.2.6 by Robert Collins
Merge bzr.dev
27
    int ENOTDIR
28
    int EAGAIN
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
29
    int errno
30
    char *strerror(int errno)
31
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
32
cdef extern from 'unistd.h':
33
    int chdir(char *path)
34
    char *getcwd(char *, int size)
35
36
cdef extern from 'stdlib.h':
37
    void *malloc(int)
38
    void free(void *)
39
3696.3.10 by Robert Collins
Review feedback.
40
41
cdef extern from 'sys/types.h':
42
    ctypedef long ssize_t
43
    ctypedef unsigned long size_t
44
    ctypedef long time_t
45
    ctypedef unsigned long ino_t
46
    ctypedef unsigned long long off_t
47
48
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
49
cdef extern from 'sys/stat.h':
50
    cdef struct stat:
51
        int st_mode
3696.3.10 by Robert Collins
Review feedback.
52
        off_t st_size
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
53
        int st_dev
3696.3.10 by Robert Collins
Review feedback.
54
        ino_t st_ino
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
55
        int st_mtime
56
        int st_ctime
57
    int lstat(char *path, stat *buf)
58
    int S_ISDIR(int mode)
59
    int S_ISCHR(int mode)
60
    int S_ISBLK(int mode)
61
    int S_ISREG(int mode)
62
    int S_ISFIFO(int mode)
63
    int S_ISLNK(int mode)
64
    int S_ISSOCK(int mode)
65
66
67
cdef extern from 'Python.h':
68
    char * PyString_AS_STRING(object)
69
    ctypedef int Py_ssize_t # Required for older pyrex versions
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
70
    ctypedef struct PyObject:
71
        pass
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
72
    Py_ssize_t PyString_Size(object s)
73
    object PyList_GetItem(object lst, Py_ssize_t index)
74
    void *PyList_GetItem_object_void "PyList_GET_ITEM" (object lst, int index)
75
    int PyList_Append(object lst, object item) except -1
76
    void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index)
77
    int PyTuple_SetItem(void *, Py_ssize_t pos, object item) except -1
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
78
    int PyTuple_SetItem_obj "PyTuple_SetItem" (void *, Py_ssize_t pos, PyObject * item) except -1
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
79
    void Py_INCREF(object o)
80
    void Py_DECREF(object o)
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
81
    void PyString_Concat(PyObject **string, object newpart)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
82
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
83
84
cdef extern from 'dirent.h':
85
    ctypedef struct dirent:
86
        char d_name[256]
3696.3.6 by Robert Collins
Partial review feedback fixups.
87
        ino_t d_ino
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
88
    ctypedef struct DIR
89
    # should be DIR *, pyrex barfs.
1739.2.6 by Robert Collins
Merge bzr.dev
90
    DIR * opendir(char * name)
91
    int closedir(DIR * dir)
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
92
    dirent *readdir(DIR *dir)
93
94
_directory = 'directory'
95
_chardev = 'chardev'
96
_block = 'block'
97
_file = 'file'
98
_fifo = 'fifo'
99
_symlink = 'symlink'
100
_socket = 'socket'
101
_unknown = 'unknown'
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
102
_missing = 'missing'
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
103
104
# add a typedef struct dirent dirent to workaround pyrex
105
cdef extern from 'readdir.h':
106
    pass
107
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
108
109
cdef class _Stat:
110
    """Represent a 'stat' result."""
111
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
112
    cdef stat _st
113
114
    property st_dev:
115
        def __get__(self):
116
            return self._st.st_dev
117
118
    property st_ino:
119
        def __get__(self):
120
            return self._st.st_ino
121
122
    property st_mode:
123
        def __get__(self):
124
            return self._st.st_mode
125
126
    property st_ctime:
127
        def __get__(self):
128
            return self._st.st_ctime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
129
130
    property st_mtime:
131
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
132
            return self._st.st_mtime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
133
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
134
    property st_size:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
135
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
136
            return self._st.st_size
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
137
138
    def __repr__(self):
139
        """Repr is the same as a Stat object.
140
3696.3.6 by Robert Collins
Partial review feedback fixups.
141
        (mode, ino, dev, nlink, uid, gid, size, None(atime), mtime, ctime)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
142
        """
143
        return repr((self.st_mode, 0, 0, 0, 0, 0, self.st_size, None,
144
                     self._mtime, self._ctime))
145
146
147
from bzrlib import osutils
148
149
150
cdef class UTF8DirReader:
151
    """A dir reader for utf8 file systems."""
152
153
    cdef readonly object _safe_utf8
154
    cdef _directory, _chardev, _block, _file, _fifo, _symlink
155
    cdef _socket, _unknown
156
157
    def __init__(self):
158
        self._safe_utf8 = osutils.safe_utf8
159
        self._directory = _directory
160
        self._chardev = _chardev
161
        self._block = _block
162
        self._file = _file
163
        self._fifo = _fifo
164
        self._symlink = _symlink
165
        self._socket = _socket
166
        self._unknown = _unknown
167
168
    def kind_from_mode(self, int mode):
169
        """Get the kind of a path from a mode status."""
170
        return self._kind_from_mode(mode)
171
172
    cdef _kind_from_mode(self, int mode):
3696.3.10 by Robert Collins
Review feedback.
173
        # Files and directories are the most common - check them first.
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
174
        if S_ISREG(mode):
175
            return self._file
176
        if S_ISDIR(mode):
177
            return self._directory
178
        if S_ISCHR(mode):
179
            return self._chardev
180
        if S_ISBLK(mode):
181
            return self._block
182
        if S_ISLNK(mode):
183
            return self._symlink
184
        if S_ISFIFO(mode):
185
            return self._fifo
186
        if S_ISSOCK(mode):
187
            return self._socket
188
        return self._unknown
189
190
    def top_prefix_to_starting_dir(self, top, prefix=""):
191
        """See DirReader.top_prefix_to_starting_dir."""
192
        return (self._safe_utf8(prefix), None, None, None,
193
            self._safe_utf8(top))
194
195
    def read_dir(self, prefix, top):
196
        """Read a single directory from a utf8 file system.
197
198
        All paths in and out are utf8.
199
200
        This sub-function is called when we know the filesystem is already in utf8
201
        encoding. So we don't need to transcode filenames.
202
203
        See DirReader.read_dir for details.
204
        """
205
        #cdef char *_prefix = prefix
206
        #cdef char *_top = top
207
        # Use C accelerated directory listing.
208
        cdef object newval
209
        cdef int index
210
        cdef int length
211
        cdef void * atuple
212
        cdef object name
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
213
        cdef PyObject * new_val_obj
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
214
215
        if PyString_Size(prefix):
216
            relprefix = prefix + '/'
217
        else:
218
            relprefix = ''
219
        top_slash = top + '/'
220
221
        # read_dir supplies in should-stat order.
222
        # for _, name in sorted(_listdir(top)):
223
        result = _read_dir(top)
224
        length = len(result)
225
        # result.sort()
226
        for index from 0 <= index < length:
227
            atuple = PyList_GetItem_object_void(result, index)
228
            name = <object>PyTuple_GetItem_void_void(atuple, 1)
3696.3.10 by Robert Collins
Review feedback.
229
            # We have a tuple with (inode, name, None, statvalue, None)
230
            # Now edit it:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
231
            # inode -> path_from_top
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
232
            # direct concat - faster than operator +.
233
            new_val_obj = <PyObject *>relprefix
234
            Py_INCREF(relprefix)
235
            PyString_Concat(&new_val_obj, name)
236
            if NULL == new_val_obj:
237
                # PyString_Concat will have setup an exception, but how to get
238
                # at it?
239
                raise Exception("failed to strcat")
240
            PyTuple_SetItem_obj(atuple, 0, new_val_obj)
3696.3.10 by Robert Collins
Review feedback.
241
            # 1st None -> kind
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
242
            newval = self._kind_from_mode(
243
                (<_Stat>PyTuple_GetItem_void_void(atuple, 3)).st_mode)
244
            Py_INCREF(newval)
245
            PyTuple_SetItem(atuple, 2, newval)
3696.3.10 by Robert Collins
Review feedback.
246
            # 2nd None -> abspath # for all - the caller may need to stat files
247
            # etc.
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
248
            # direct concat - faster than operator +.
249
            new_val_obj = <PyObject *>top_slash
250
            Py_INCREF(top_slash)
251
            PyString_Concat(&new_val_obj, name)
252
            if NULL == new_val_obj:
253
                # PyString_Concat will have setup an exception, but how to get
254
                # at it?
255
                raise Exception("failed to strcat")
256
            PyTuple_SetItem_obj(atuple, 4, new_val_obj)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
257
        return result
258
259
260
cdef _read_dir(path):
1739.2.11 by Robert Collins
Docstring and copyright header update per Martin's review.
261
    """Like os.listdir, this reads the contents of a directory.
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
262
263
    :param path: the directory to list.
3696.3.10 by Robert Collins
Review feedback.
264
    :return: a list of single-owner (the list) tuples ready for editing into
265
        the result tuples walkdirs needs to yield. They contain (inode, name,
266
        None, statvalue, None).
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
267
    """
268
    cdef DIR *the_dir
269
    # currently this needs a fixup - the C code says 'dirent' but should say
270
    # 'struct dirent'
271
    cdef dirent * entry
1739.2.6 by Robert Collins
Merge bzr.dev
272
    cdef dirent sentinel
273
    cdef char *name
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
274
    cdef int stat_result
275
    cdef _Stat statvalue
276
    cdef char *cwd
277
278
    cwd = getcwd(NULL, 0)
279
    if -1 == chdir(path):
280
        raise OSError(errno, strerror(errno))
281
    the_dir = opendir(".")
1739.2.6 by Robert Collins
Merge bzr.dev
282
    if NULL == the_dir:
283
        raise OSError(errno, strerror(errno))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
284
    result = []
285
    try:
1739.2.6 by Robert Collins
Merge bzr.dev
286
        entry = &sentinel
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
287
        while entry != NULL:
1739.2.6 by Robert Collins
Merge bzr.dev
288
            entry = readdir(the_dir)
289
            if entry == NULL:
290
                if errno == EAGAIN:
291
                    # try again
292
                    continue
293
                elif errno != ENOTDIR and errno != ENOENT and errno != 0:
294
                    # We see ENOTDIR at the end of a normal directory.
295
                    # As ENOTDIR for read_dir(file) is triggered on opendir,
296
                    # we consider ENOTDIR to be 'no error'.
297
                    # ENOENT is listed as 'invalid position in the dir stream' for
298
                    # readdir. We swallow this for now and just keep reading.
299
                    raise OSError(errno, strerror(errno))
300
                else:
301
                    # done
302
                    continue
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
303
            name = entry.d_name
3696.3.6 by Robert Collins
Partial review feedback fixups.
304
            if not (name[0] == c"." and (
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
305
                (name[1] == 0) or 
3696.3.6 by Robert Collins
Partial review feedback fixups.
306
                (name[1] == c"." and name[2] == 0))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
307
                ):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
308
                statvalue = _Stat()
309
                stat_result = lstat(entry.d_name, &statvalue._st)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
310
                if stat_result != 0:
311
                    if errno != ENOENT:
312
                        raise OSError(errno, strerror(errno))
313
                    else:
314
                        kind = _missing
315
                        statvalue = None
316
                # We append a 5-tuple that can be modified in-place by the C
317
                # api:
3696.3.6 by Robert Collins
Partial review feedback fixups.
318
                # inode to sort on (to replace with top_path)
319
                # name (to keep)
320
                # kind (None, to set)
321
                # statvalue (to keep)
322
                # abspath (None, to set)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
323
                PyList_Append(result, (entry.d_ino, entry.d_name, None,
324
                    statvalue, None))
1739.2.6 by Robert Collins
Merge bzr.dev
325
    finally:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
326
        if -1 == chdir(cwd):
327
            free(cwd)
328
            raise OSError(errno, strerror(errno))
329
        free(cwd)
1739.2.6 by Robert Collins
Merge bzr.dev
330
        if -1 == closedir(the_dir):
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
331
            raise OSError(errno, strerror(errno))
332
    return result
1739.2.6 by Robert Collins
Merge bzr.dev
333
334
335
# vim: tw=79 ai expandtab sw=4 sts=4