bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
1739.2.6
by Robert Collins
Merge bzr.dev |
1 |
# Copyright (C) 2006, 2008 Canonical Ltd
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
||
1739.2.7
by Robert Collins
Update readdir pyrex source files and usage in line with current practice. |
17 |
"""Wrapper for readdir which returns files ordered by inode."""
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
18 |
|
19 |
||
20 |
import os |
|
21 |
import sys |
|
22 |
||
23 |
||
24 |
# the opaque C library DIR type.
|
|
25 |
cdef extern from 'errno.h': |
|
26 |
int ENOENT |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
27 |
int ENOTDIR |
28 |
int EAGAIN |
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
29 |
int errno |
30 |
char *strerror(int errno) |
|
31 |
||
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
32 |
cdef extern from 'unistd.h': |
33 |
int chdir(char *path) |
|
34 |
char *getcwd(char *, int size) |
|
35 |
||
36 |
cdef extern from 'stdlib.h': |
|
37 |
void *malloc(int) |
|
38 |
void free(void *) |
|
39 |
||
40 |
cdef extern from 'sys/stat.h': |
|
41 |
cdef struct stat: |
|
42 |
int st_mode |
|
43 |
int st_size |
|
44 |
int st_dev |
|
45 |
int st_ino |
|
46 |
int st_mtime |
|
47 |
int st_ctime |
|
48 |
int lstat(char *path, stat *buf) |
|
49 |
int S_ISDIR(int mode) |
|
50 |
int S_ISCHR(int mode) |
|
51 |
int S_ISBLK(int mode) |
|
52 |
int S_ISREG(int mode) |
|
53 |
int S_ISFIFO(int mode) |
|
54 |
int S_ISLNK(int mode) |
|
55 |
int S_ISSOCK(int mode) |
|
56 |
||
57 |
||
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
58 |
cdef extern from 'sys/types.h': |
59 |
ctypedef long ssize_t |
|
60 |
ctypedef unsigned long size_t |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
61 |
ctypedef long time_t |
3696.3.6
by Robert Collins
Partial review feedback fixups. |
62 |
ctypedef unsigned long ino_t |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
63 |
|
64 |
||
65 |
cdef extern from 'Python.h': |
|
66 |
char * PyString_AS_STRING(object) |
|
67 |
ctypedef int Py_ssize_t # Required for older pyrex versions |
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
68 |
ctypedef struct PyObject: |
69 |
pass
|
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
70 |
Py_ssize_t PyString_Size(object s) |
71 |
object PyList_GetItem(object lst, Py_ssize_t index) |
|
72 |
void *PyList_GetItem_object_void "PyList_GET_ITEM" (object lst, int index) |
|
73 |
int PyList_Append(object lst, object item) except -1 |
|
74 |
void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index) |
|
75 |
int PyTuple_SetItem(void *, Py_ssize_t pos, object item) except -1 |
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
76 |
int PyTuple_SetItem_obj "PyTuple_SetItem" (void *, Py_ssize_t pos, PyObject * item) except -1 |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
77 |
void Py_INCREF(object o) |
78 |
void Py_DECREF(object o) |
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
79 |
void PyString_Concat(PyObject **string, object newpart) |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
80 |
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
81 |
|
82 |
cdef extern from 'dirent.h': |
|
83 |
ctypedef struct dirent: |
|
84 |
char d_name[256] |
|
3696.3.6
by Robert Collins
Partial review feedback fixups. |
85 |
ino_t d_ino |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
86 |
ctypedef struct DIR |
87 |
# should be DIR *, pyrex barfs.
|
|
1739.2.6
by Robert Collins
Merge bzr.dev |
88 |
DIR * opendir(char * name) |
89 |
int closedir(DIR * dir) |
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
90 |
dirent *readdir(DIR *dir) |
91 |
||
92 |
_directory = 'directory' |
|
93 |
_chardev = 'chardev' |
|
94 |
_block = 'block' |
|
95 |
_file = 'file' |
|
96 |
_fifo = 'fifo' |
|
97 |
_symlink = 'symlink' |
|
98 |
_socket = 'socket' |
|
99 |
_unknown = 'unknown' |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
100 |
_missing = 'missing' |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
101 |
|
102 |
# add a typedef struct dirent dirent to workaround pyrex
|
|
103 |
cdef extern from 'readdir.h': |
|
104 |
pass
|
|
105 |
||
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
106 |
|
107 |
cdef class _Stat: |
|
108 |
"""Represent a 'stat' result.""" |
|
109 |
||
3696.3.8
by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always. |
110 |
cdef stat _st |
111 |
||
112 |
property st_dev: |
|
113 |
def __get__(self): |
|
114 |
return self._st.st_dev |
|
115 |
||
116 |
property st_ino: |
|
117 |
def __get__(self): |
|
118 |
return self._st.st_ino |
|
119 |
||
120 |
property st_mode: |
|
121 |
def __get__(self): |
|
122 |
return self._st.st_mode |
|
123 |
||
124 |
property st_ctime: |
|
125 |
def __get__(self): |
|
126 |
return self._st.st_ctime |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
127 |
|
128 |
property st_mtime: |
|
129 |
def __get__(self): |
|
3696.3.8
by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always. |
130 |
return self._st.st_mtime |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
131 |
|
3696.3.8
by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always. |
132 |
property st_size: |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
133 |
def __get__(self): |
3696.3.8
by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always. |
134 |
return self._st.st_size |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
135 |
|
136 |
def __repr__(self): |
|
137 |
"""Repr is the same as a Stat object. |
|
138 |
||
3696.3.6
by Robert Collins
Partial review feedback fixups. |
139 |
(mode, ino, dev, nlink, uid, gid, size, None(atime), mtime, ctime)
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
140 |
"""
|
141 |
return repr((self.st_mode, 0, 0, 0, 0, 0, self.st_size, None, |
|
142 |
self._mtime, self._ctime)) |
|
143 |
||
144 |
||
145 |
from bzrlib import osutils |
|
146 |
||
147 |
||
148 |
cdef class UTF8DirReader: |
|
149 |
"""A dir reader for utf8 file systems.""" |
|
150 |
||
151 |
cdef readonly object _safe_utf8 |
|
152 |
cdef _directory, _chardev, _block, _file, _fifo, _symlink |
|
153 |
cdef _socket, _unknown |
|
154 |
||
155 |
def __init__(self): |
|
156 |
self._safe_utf8 = osutils.safe_utf8 |
|
157 |
self._directory = _directory |
|
158 |
self._chardev = _chardev |
|
159 |
self._block = _block |
|
160 |
self._file = _file |
|
161 |
self._fifo = _fifo |
|
162 |
self._symlink = _symlink |
|
163 |
self._socket = _socket |
|
164 |
self._unknown = _unknown |
|
165 |
||
166 |
def kind_from_mode(self, int mode): |
|
167 |
"""Get the kind of a path from a mode status.""" |
|
168 |
return self._kind_from_mode(mode) |
|
169 |
||
170 |
cdef _kind_from_mode(self, int mode): |
|
171 |
# in order of frequency:
|
|
172 |
if S_ISREG(mode): |
|
173 |
return self._file |
|
174 |
if S_ISDIR(mode): |
|
175 |
return self._directory |
|
176 |
if S_ISCHR(mode): |
|
177 |
return self._chardev |
|
178 |
if S_ISBLK(mode): |
|
179 |
return self._block |
|
180 |
if S_ISLNK(mode): |
|
181 |
return self._symlink |
|
182 |
if S_ISFIFO(mode): |
|
183 |
return self._fifo |
|
184 |
if S_ISSOCK(mode): |
|
185 |
return self._socket |
|
186 |
return self._unknown |
|
187 |
||
188 |
def top_prefix_to_starting_dir(self, top, prefix=""): |
|
189 |
"""See DirReader.top_prefix_to_starting_dir.""" |
|
190 |
return (self._safe_utf8(prefix), None, None, None, |
|
191 |
self._safe_utf8(top)) |
|
192 |
||
193 |
def read_dir(self, prefix, top): |
|
194 |
"""Read a single directory from a utf8 file system. |
|
195 |
||
196 |
All paths in and out are utf8.
|
|
197 |
||
198 |
This sub-function is called when we know the filesystem is already in utf8
|
|
199 |
encoding. So we don't need to transcode filenames.
|
|
200 |
||
201 |
See DirReader.read_dir for details.
|
|
202 |
"""
|
|
203 |
#cdef char *_prefix = prefix
|
|
204 |
#cdef char *_top = top
|
|
205 |
# Use C accelerated directory listing.
|
|
206 |
cdef object newval |
|
207 |
cdef int index |
|
208 |
cdef int length |
|
209 |
cdef void * atuple |
|
210 |
cdef object name |
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
211 |
cdef PyObject * new_val_obj |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
212 |
|
213 |
if PyString_Size(prefix): |
|
214 |
relprefix = prefix + '/' |
|
215 |
else: |
|
216 |
relprefix = '' |
|
217 |
top_slash = top + '/' |
|
218 |
||
219 |
# read_dir supplies in should-stat order.
|
|
220 |
# for _, name in sorted(_listdir(top)):
|
|
221 |
result = _read_dir(top) |
|
222 |
length = len(result) |
|
223 |
# result.sort()
|
|
224 |
for index from 0 <= index < length: |
|
225 |
atuple = PyList_GetItem_object_void(result, index) |
|
226 |
name = <object>PyTuple_GetItem_void_void(atuple, 1) |
|
227 |
# We have inode, name, None, statvalue, None
|
|
228 |
# inode -> path_from_top
|
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
229 |
# direct concat - faster than operator +.
|
230 |
new_val_obj = <PyObject *>relprefix |
|
231 |
Py_INCREF(relprefix) |
|
232 |
PyString_Concat(&new_val_obj, name) |
|
233 |
if NULL == new_val_obj: |
|
234 |
# PyString_Concat will have setup an exception, but how to get
|
|
235 |
# at it?
|
|
236 |
raise Exception("failed to strcat") |
|
237 |
PyTuple_SetItem_obj(atuple, 0, new_val_obj) |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
238 |
# None -> kind
|
239 |
newval = self._kind_from_mode( |
|
240 |
(<_Stat>PyTuple_GetItem_void_void(atuple, 3)).st_mode) |
|
241 |
Py_INCREF(newval) |
|
242 |
PyTuple_SetItem(atuple, 2, newval) |
|
243 |
# none -> abspath # perhaps only do if its a dir?
|
|
3696.3.7
by Robert Collins
Use PyString_Concat directly for another small boost. |
244 |
# direct concat - faster than operator +.
|
245 |
new_val_obj = <PyObject *>top_slash |
|
246 |
Py_INCREF(top_slash) |
|
247 |
PyString_Concat(&new_val_obj, name) |
|
248 |
if NULL == new_val_obj: |
|
249 |
# PyString_Concat will have setup an exception, but how to get
|
|
250 |
# at it?
|
|
251 |
raise Exception("failed to strcat") |
|
252 |
PyTuple_SetItem_obj(atuple, 4, new_val_obj) |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
253 |
return result |
254 |
||
255 |
||
256 |
cdef _read_dir(path): |
|
1739.2.11
by Robert Collins
Docstring and copyright header update per Martin's review. |
257 |
"""Like os.listdir, this reads the contents of a directory. |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
258 |
|
259 |
:param path: the directory to list.
|
|
1739.2.7
by Robert Collins
Update readdir pyrex source files and usage in line with current practice. |
260 |
:return: a list of (sort_key, basename) tuples.
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
261 |
"""
|
262 |
cdef DIR *the_dir |
|
263 |
# currently this needs a fixup - the C code says 'dirent' but should say
|
|
264 |
# 'struct dirent'
|
|
265 |
cdef dirent * entry |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
266 |
cdef dirent sentinel |
267 |
cdef char *name |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
268 |
cdef int stat_result |
269 |
cdef _Stat statvalue |
|
270 |
cdef char *cwd |
|
271 |
||
272 |
cwd = getcwd(NULL, 0) |
|
273 |
if -1 == chdir(path): |
|
274 |
raise OSError(errno, strerror(errno)) |
|
275 |
the_dir = opendir(".") |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
276 |
if NULL == the_dir: |
277 |
raise OSError(errno, strerror(errno)) |
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
278 |
result = [] |
279 |
try: |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
280 |
entry = &sentinel |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
281 |
while entry != NULL: |
1739.2.6
by Robert Collins
Merge bzr.dev |
282 |
entry = readdir(the_dir) |
283 |
if entry == NULL: |
|
284 |
if errno == EAGAIN: |
|
285 |
# try again
|
|
286 |
continue
|
|
287 |
elif errno != ENOTDIR and errno != ENOENT and errno != 0: |
|
288 |
# We see ENOTDIR at the end of a normal directory.
|
|
289 |
# As ENOTDIR for read_dir(file) is triggered on opendir,
|
|
290 |
# we consider ENOTDIR to be 'no error'.
|
|
291 |
# ENOENT is listed as 'invalid position in the dir stream' for
|
|
292 |
# readdir. We swallow this for now and just keep reading.
|
|
293 |
raise OSError(errno, strerror(errno)) |
|
294 |
else: |
|
295 |
# done
|
|
296 |
continue
|
|
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
297 |
name = entry.d_name |
3696.3.6
by Robert Collins
Partial review feedback fixups. |
298 |
if not (name[0] == c"." and ( |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
299 |
(name[1] == 0) or |
3696.3.6
by Robert Collins
Partial review feedback fixups. |
300 |
(name[1] == c"." and name[2] == 0)) |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
301 |
):
|
3696.3.8
by Robert Collins
Just embed a struct st in the python result object, avoids converting things we don't need converted, and copying values around always. |
302 |
statvalue = _Stat() |
303 |
stat_result = lstat(entry.d_name, &statvalue._st) |
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
304 |
if stat_result != 0: |
305 |
if errno != ENOENT: |
|
306 |
raise OSError(errno, strerror(errno)) |
|
307 |
else: |
|
308 |
kind = _missing |
|
309 |
statvalue = None |
|
310 |
# We append a 5-tuple that can be modified in-place by the C
|
|
311 |
# api:
|
|
3696.3.6
by Robert Collins
Partial review feedback fixups. |
312 |
# inode to sort on (to replace with top_path)
|
313 |
# name (to keep)
|
|
314 |
# kind (None, to set)
|
|
315 |
# statvalue (to keep)
|
|
316 |
# abspath (None, to set)
|
|
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
317 |
PyList_Append(result, (entry.d_ino, entry.d_name, None, |
318 |
statvalue, None)) |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
319 |
finally: |
3696.3.5
by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins) |
320 |
if -1 == chdir(cwd): |
321 |
free(cwd) |
|
322 |
raise OSError(errno, strerror(errno)) |
|
323 |
free(cwd) |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
324 |
if -1 == closedir(the_dir): |
1739.2.3
by Robert Collins
Add a replacement for os.listdir which returns file kind information from readdir when it is available. This drops our osutils.walkdirs time further, down to 77ms. |
325 |
raise OSError(errno, strerror(errno)) |
326 |
return result |
|
1739.2.6
by Robert Collins
Merge bzr.dev |
327 |
|
328 |
||
329 |
# vim: tw=79 ai expandtab sw=4 sts=4
|