/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
1739.2.6 by Robert Collins
Merge bzr.dev
1
# Copyright (C) 2006, 2008 Canonical Ltd
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
17
"""Wrapper for readdir which returns files ordered by inode."""
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
18
19
20
import os
21
import sys
22
23
24
# the opaque C library DIR type.
25
cdef extern from 'errno.h':
26
    int ENOENT
1739.2.6 by Robert Collins
Merge bzr.dev
27
    int ENOTDIR
28
    int EAGAIN
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
29
    int errno
30
    char *strerror(int errno)
31
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
32
cdef extern from 'unistd.h':
33
    int chdir(char *path)
34
    char *getcwd(char *, int size)
35
36
cdef extern from 'stdlib.h':
37
    void *malloc(int)
38
    void free(void *)
39
40
cdef extern from 'sys/stat.h':
41
    cdef struct stat:
42
        int st_mode
43
        int st_size
44
        int st_dev
45
        int st_ino
46
        int st_mtime
47
        int st_ctime
48
    int lstat(char *path, stat *buf)
49
    int S_ISDIR(int mode)
50
    int S_ISCHR(int mode)
51
    int S_ISBLK(int mode)
52
    int S_ISREG(int mode)
53
    int S_ISFIFO(int mode)
54
    int S_ISLNK(int mode)
55
    int S_ISSOCK(int mode)
56
57
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
58
cdef extern from 'sys/types.h':
59
    ctypedef long ssize_t
60
    ctypedef unsigned long size_t
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
61
    ctypedef long time_t
3696.3.6 by Robert Collins
Partial review feedback fixups.
62
    ctypedef unsigned long ino_t
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
63
64
65
cdef extern from 'Python.h':
66
    char * PyString_AS_STRING(object)
67
    ctypedef int Py_ssize_t # Required for older pyrex versions
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
68
    ctypedef struct PyObject:
69
        pass
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
70
    Py_ssize_t PyString_Size(object s)
71
    object PyList_GetItem(object lst, Py_ssize_t index)
72
    void *PyList_GetItem_object_void "PyList_GET_ITEM" (object lst, int index)
73
    int PyList_Append(object lst, object item) except -1
74
    void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index)
75
    int PyTuple_SetItem(void *, Py_ssize_t pos, object item) except -1
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
76
    int PyTuple_SetItem_obj "PyTuple_SetItem" (void *, Py_ssize_t pos, PyObject * item) except -1
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
77
    void Py_INCREF(object o)
78
    void Py_DECREF(object o)
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
79
    void PyString_Concat(PyObject **string, object newpart)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
80
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
81
82
cdef extern from 'dirent.h':
83
    ctypedef struct dirent:
84
        char d_name[256]
3696.3.6 by Robert Collins
Partial review feedback fixups.
85
        ino_t d_ino
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
86
    ctypedef struct DIR
87
    # should be DIR *, pyrex barfs.
1739.2.6 by Robert Collins
Merge bzr.dev
88
    DIR * opendir(char * name)
89
    int closedir(DIR * dir)
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
90
    dirent *readdir(DIR *dir)
91
92
_directory = 'directory'
93
_chardev = 'chardev'
94
_block = 'block'
95
_file = 'file'
96
_fifo = 'fifo'
97
_symlink = 'symlink'
98
_socket = 'socket'
99
_unknown = 'unknown'
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
100
_missing = 'missing'
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
101
102
# add a typedef struct dirent dirent to workaround pyrex
103
cdef extern from 'readdir.h':
104
    pass
105
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
106
107
cdef class _Stat:
108
    """Represent a 'stat' result."""
109
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
110
    cdef stat _st
111
112
    property st_dev:
113
        def __get__(self):
114
            return self._st.st_dev
115
116
    property st_ino:
117
        def __get__(self):
118
            return self._st.st_ino
119
120
    property st_mode:
121
        def __get__(self):
122
            return self._st.st_mode
123
124
    property st_ctime:
125
        def __get__(self):
126
            return self._st.st_ctime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
127
128
    property st_mtime:
129
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
130
            return self._st.st_mtime
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
131
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
132
    property st_size:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
133
        def __get__(self):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
134
            return self._st.st_size
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
135
136
    def __repr__(self):
137
        """Repr is the same as a Stat object.
138
3696.3.6 by Robert Collins
Partial review feedback fixups.
139
        (mode, ino, dev, nlink, uid, gid, size, None(atime), mtime, ctime)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
140
        """
141
        return repr((self.st_mode, 0, 0, 0, 0, 0, self.st_size, None,
142
                     self._mtime, self._ctime))
143
144
145
from bzrlib import osutils
146
147
148
cdef class UTF8DirReader:
149
    """A dir reader for utf8 file systems."""
150
151
    cdef readonly object _safe_utf8
152
    cdef _directory, _chardev, _block, _file, _fifo, _symlink
153
    cdef _socket, _unknown
154
155
    def __init__(self):
156
        self._safe_utf8 = osutils.safe_utf8
157
        self._directory = _directory
158
        self._chardev = _chardev
159
        self._block = _block
160
        self._file = _file
161
        self._fifo = _fifo
162
        self._symlink = _symlink
163
        self._socket = _socket
164
        self._unknown = _unknown
165
166
    def kind_from_mode(self, int mode):
167
        """Get the kind of a path from a mode status."""
168
        return self._kind_from_mode(mode)
169
170
    cdef _kind_from_mode(self, int mode):
171
        # in order of frequency:
172
        if S_ISREG(mode):
173
            return self._file
174
        if S_ISDIR(mode):
175
            return self._directory
176
        if S_ISCHR(mode):
177
            return self._chardev
178
        if S_ISBLK(mode):
179
            return self._block
180
        if S_ISLNK(mode):
181
            return self._symlink
182
        if S_ISFIFO(mode):
183
            return self._fifo
184
        if S_ISSOCK(mode):
185
            return self._socket
186
        return self._unknown
187
188
    def top_prefix_to_starting_dir(self, top, prefix=""):
189
        """See DirReader.top_prefix_to_starting_dir."""
190
        return (self._safe_utf8(prefix), None, None, None,
191
            self._safe_utf8(top))
192
193
    def read_dir(self, prefix, top):
194
        """Read a single directory from a utf8 file system.
195
196
        All paths in and out are utf8.
197
198
        This sub-function is called when we know the filesystem is already in utf8
199
        encoding. So we don't need to transcode filenames.
200
201
        See DirReader.read_dir for details.
202
        """
203
        #cdef char *_prefix = prefix
204
        #cdef char *_top = top
205
        # Use C accelerated directory listing.
206
        cdef object newval
207
        cdef int index
208
        cdef int length
209
        cdef void * atuple
210
        cdef object name
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
211
        cdef PyObject * new_val_obj
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
212
213
        if PyString_Size(prefix):
214
            relprefix = prefix + '/'
215
        else:
216
            relprefix = ''
217
        top_slash = top + '/'
218
219
        # read_dir supplies in should-stat order.
220
        # for _, name in sorted(_listdir(top)):
221
        result = _read_dir(top)
222
        length = len(result)
223
        # result.sort()
224
        for index from 0 <= index < length:
225
            atuple = PyList_GetItem_object_void(result, index)
226
            name = <object>PyTuple_GetItem_void_void(atuple, 1)
227
            # We have inode, name, None, statvalue, None
228
            # inode -> path_from_top
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
229
            # direct concat - faster than operator +.
230
            new_val_obj = <PyObject *>relprefix
231
            Py_INCREF(relprefix)
232
            PyString_Concat(&new_val_obj, name)
233
            if NULL == new_val_obj:
234
                # PyString_Concat will have setup an exception, but how to get
235
                # at it?
236
                raise Exception("failed to strcat")
237
            PyTuple_SetItem_obj(atuple, 0, new_val_obj)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
238
            # None -> kind
239
            newval = self._kind_from_mode(
240
                (<_Stat>PyTuple_GetItem_void_void(atuple, 3)).st_mode)
241
            Py_INCREF(newval)
242
            PyTuple_SetItem(atuple, 2, newval)
243
            # none -> abspath # perhaps only do if its a dir?
3696.3.7 by Robert Collins
Use PyString_Concat directly for another small boost.
244
            # direct concat - faster than operator +.
245
            new_val_obj = <PyObject *>top_slash
246
            Py_INCREF(top_slash)
247
            PyString_Concat(&new_val_obj, name)
248
            if NULL == new_val_obj:
249
                # PyString_Concat will have setup an exception, but how to get
250
                # at it?
251
                raise Exception("failed to strcat")
252
            PyTuple_SetItem_obj(atuple, 4, new_val_obj)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
253
        return result
254
255
256
cdef _read_dir(path):
1739.2.11 by Robert Collins
Docstring and copyright header update per Martin's review.
257
    """Like os.listdir, this reads the contents of a directory.
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
258
259
    :param path: the directory to list.
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
260
    :return: a list of (sort_key, basename) tuples.
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
261
    """
262
    cdef DIR *the_dir
263
    # currently this needs a fixup - the C code says 'dirent' but should say
264
    # 'struct dirent'
265
    cdef dirent * entry
1739.2.6 by Robert Collins
Merge bzr.dev
266
    cdef dirent sentinel
267
    cdef char *name
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
268
    cdef int stat_result
269
    cdef _Stat statvalue
270
    cdef char *cwd
271
272
    cwd = getcwd(NULL, 0)
273
    if -1 == chdir(path):
274
        raise OSError(errno, strerror(errno))
275
    the_dir = opendir(".")
1739.2.6 by Robert Collins
Merge bzr.dev
276
    if NULL == the_dir:
277
        raise OSError(errno, strerror(errno))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
278
    result = []
279
    try:
1739.2.6 by Robert Collins
Merge bzr.dev
280
        entry = &sentinel
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
281
        while entry != NULL:
1739.2.6 by Robert Collins
Merge bzr.dev
282
            entry = readdir(the_dir)
283
            if entry == NULL:
284
                if errno == EAGAIN:
285
                    # try again
286
                    continue
287
                elif errno != ENOTDIR and errno != ENOENT and errno != 0:
288
                    # We see ENOTDIR at the end of a normal directory.
289
                    # As ENOTDIR for read_dir(file) is triggered on opendir,
290
                    # we consider ENOTDIR to be 'no error'.
291
                    # ENOENT is listed as 'invalid position in the dir stream' for
292
                    # readdir. We swallow this for now and just keep reading.
293
                    raise OSError(errno, strerror(errno))
294
                else:
295
                    # done
296
                    continue
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
297
            name = entry.d_name
3696.3.6 by Robert Collins
Partial review feedback fixups.
298
            if not (name[0] == c"." and (
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
299
                (name[1] == 0) or 
3696.3.6 by Robert Collins
Partial review feedback fixups.
300
                (name[1] == c"." and name[2] == 0))
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
301
                ):
3696.3.8 by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always.
302
                statvalue = _Stat()
303
                stat_result = lstat(entry.d_name, &statvalue._st)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
304
                if stat_result != 0:
305
                    if errno != ENOENT:
306
                        raise OSError(errno, strerror(errno))
307
                    else:
308
                        kind = _missing
309
                        statvalue = None
310
                # We append a 5-tuple that can be modified in-place by the C
311
                # api:
3696.3.6 by Robert Collins
Partial review feedback fixups.
312
                # inode to sort on (to replace with top_path)
313
                # name (to keep)
314
                # kind (None, to set)
315
                # statvalue (to keep)
316
                # abspath (None, to set)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
317
                PyList_Append(result, (entry.d_ino, entry.d_name, None,
318
                    statvalue, None))
1739.2.6 by Robert Collins
Merge bzr.dev
319
    finally:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
320
        if -1 == chdir(cwd):
321
            free(cwd)
322
            raise OSError(errno, strerror(errno))
323
        free(cwd)
1739.2.6 by Robert Collins
Merge bzr.dev
324
        if -1 == closedir(the_dir):
1739.2.3 by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms.
325
            raise OSError(errno, strerror(errno))
326
    return result
1739.2.6 by Robert Collins
Merge bzr.dev
327
328
329
# vim: tw=79 ai expandtab sw=4 sts=4