1
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
1
# Copyright (C) 2008, 2009 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
38
38
Py_ssize_t PyString_Size(object p)
39
39
Py_ssize_t PyString_GET_SIZE_ptr "PyString_GET_SIZE" (PyObject *)
40
40
char * PyString_AS_STRING_ptr "PyString_AS_STRING" (PyObject *)
41
char * PyString_AS_STRING(object)
42
Py_ssize_t PyString_GET_SIZE(object)
43
41
int PyString_AsStringAndSize_ptr(PyObject *, char **buf, Py_ssize_t *len)
44
42
void PyString_InternInPlace(PyObject **)
45
43
int PyTuple_CheckExact(object t)
57
55
# void *memrchr(void *s, int c, size_t n)
58
56
int strncmp(char *s1, char *s2, size_t n)
60
# It seems we need to import the definitions so that the pyrex compiler has
61
# local names to access them.
62
from _static_tuple_c cimport StaticTuple, \
63
import_static_tuple_c, StaticTuple_New, \
64
StaticTuple_Intern, StaticTuple_SET_ITEM, StaticTuple_CheckExact
67
59
# TODO: Find some way to import this from _dirstate_helpers
68
cdef void* _my_memrchr(void *s, int c, size_t n): # cannot_raise
60
cdef void* _my_memrchr(void *s, int c, size_t n):
69
61
# memrchr seems to be a GNU extension, so we have to implement it ourselves
70
62
# It is not present in any win32 standard library
157
143
cdef char *temp_ptr
158
144
cdef int loop_counter
161
key = StaticTuple_New(self.key_length)
146
key = PyTuple_New(self.key_length)
162
147
for loop_counter from 0 <= loop_counter < self.key_length:
163
148
# grab a key segment
164
149
temp_ptr = <char*>memchr(self._start, c'\0', last - self._start)
173
158
last - self._start)))
174
159
raise AssertionError(failure_string)
175
160
# capture the key string
176
if (self.key_length == 1
177
and (temp_ptr - self._start) == 45
178
and strncmp(self._start, 'sha1:', 5) == 0):
179
key_element = safe_string_from_size(self._start,
180
temp_ptr - self._start)
182
key_element = safe_interned_string_from_size(self._start,
161
# TODO: Consider using PyIntern_FromString, the only caveat is that
162
# it assumes a NULL-terminated string, so we have to check if
163
# temp_ptr[0] == c'\0' or some other char.
164
key_element = safe_interned_string_from_size(self._start,
183
165
temp_ptr - self._start)
184
166
# advance our pointer
185
167
self._start = temp_ptr + 1
186
168
Py_INCREF(key_element)
187
StaticTuple_SET_ITEM(key, loop_counter, key_element)
188
key = StaticTuple_Intern(key)
169
PyTuple_SET_ITEM(key, loop_counter, key_element)
191
172
cdef int process_line(self) except -1:
208
188
# And the next string is right after it
209
189
self._cur_str = last + 1
210
190
# The last character is right before the '\n'
212
193
if last == self._start:
215
196
if last < self._start:
216
197
# Unexpected error condition - fail
217
raise AssertionError("last < self._start")
218
199
if 0 == self._header_found:
219
200
# The first line in a leaf node is the header "type=leaf\n"
220
201
if strncmp("type=leaf", self._start, last - self._start) == 0:
224
205
raise AssertionError('Node did not start with "type=leaf": %r'
225
206
% (safe_string_from_size(self._start, last - self._start)))
227
209
key = self.extract_key(last)
228
210
# find the value area
229
211
temp_ptr = <char*>_my_memrchr(self._start, c'\0', last - self._start)
230
212
if temp_ptr == NULL:
232
raise AssertionError("Failed to find the value area")
234
# Because of how conversions were done, we ended up with *lots* of
235
# values that are identical. These are all of the 0-length nodes
236
# that are referred to by the TREE_ROOT (and likely some other
237
# directory nodes.) For example, bzr has 25k references to
238
# something like '12607215 328306 0 0', which ends up consuming 1MB
239
# of memory, just for those strings.
240
str_len = last - temp_ptr - 1
242
and strncmp(" 0 0", last - 4, 4) == 0):
243
# This drops peak mem for bzr.dev from 87.4MB => 86.2MB
244
# For Launchpad 236MB => 232MB
245
value = safe_interned_string_from_size(temp_ptr + 1, str_len)
247
value = safe_string_from_size(temp_ptr + 1, str_len)
216
# capture the value string
217
value = safe_string_from_size(temp_ptr + 1, last - temp_ptr - 1)
248
218
# shrink the references end point
251
220
if self.ref_list_length:
252
ref_lists = StaticTuple_New(self.ref_list_length)
254
223
while loop_counter < self.ref_list_length:
256
225
# extract a reference list
257
226
loop_counter = loop_counter + 1
258
227
if last < self._start:
259
raise AssertionError("last < self._start")
260
229
# find the next reference list end point:
261
230
temp_ptr = <char*>memchr(self._start, c'\t', last - self._start)
262
231
if temp_ptr == NULL:
263
232
# Only valid for the last list
264
233
if loop_counter != self.ref_list_length:
266
raise AssertionError(
267
"invalid key, loop_counter != self.ref_list_length")
236
raise AssertionError("invalid key")
269
238
# scan to the end of the ref list area
281
250
if temp_ptr == NULL:
282
251
# key runs to the end
283
252
temp_ptr = ref_ptr
285
253
PyList_Append(ref_list, self.extract_key(temp_ptr))
286
ref_list = StaticTuple_Intern(StaticTuple(*ref_list))
288
StaticTuple_SET_ITEM(ref_lists, loop_counter - 1, ref_list)
254
PyList_Append(ref_lists, tuple(ref_list))
289
255
# prepare for the next reference list
290
256
self._start = next_start
291
node_value = StaticTuple(value, ref_lists)
257
ref_lists = tuple(ref_lists)
258
node_value = (value, ref_lists)
293
260
if last != self._start:
294
261
# unexpected reference data present
295
raise AssertionError("unexpected reference data present")
296
node_value = StaticTuple(value, StaticTuple())
297
PyList_Append(self.keys, StaticTuple(key, node_value))
263
node_value = (value, ())
264
PyList_Append(self.keys, (key, node_value))
337
305
cdef int first_ref_list
338
306
cdef int first_reference
308
cdef PyObject *ref_bit
340
309
cdef Py_ssize_t ref_bit_len
342
if not PyTuple_CheckExact(node) and not StaticTuple_CheckExact(node):
343
raise TypeError('We expected a tuple() or StaticTuple() for node not: %s'
311
if not PyTuple_CheckExact(node):
312
raise TypeError('We expected a tuple() for node not: %s'
314
node_len = PyTuple_GET_SIZE(node)
346
315
have_reference_lists = reference_lists
347
316
if have_reference_lists:
348
317
if node_len != 4:
351
320
elif node_len < 3:
352
321
raise ValueError('Without ref_lists, we need at least 3 entries not: %s'
354
# TODO: We can probably do better than string.join(), namely
355
# when key has only 1 item, we can just grab that string
356
# And when there are 2 items, we could do a single malloc + len() + 1
357
# also, doing .join() requires a PyObject_GetAttrString call, which
358
# we could also avoid.
359
# TODO: Note that pyrex 0.9.6 generates fairly crummy code here, using the
360
# python object interface, versus 0.9.8+ which uses a helper that
361
# checks if this supports the sequence interface.
362
# We *could* do more work on our own, and grab the actual items
363
# lists. For now, just ask people to use a better compiler. :)
364
string_key = '\0'.join(node[1])
323
# I don't expect that we can do faster than string.join()
324
string_key = '\0'.join(<object>PyTuple_GET_ITEM_ptr_object(node, 1))
366
326
# TODO: instead of using string joins, precompute the final string length,
367
327
# and then malloc a single string and copy everything in.
379
339
if have_reference_lists:
380
340
# Figure out how many bytes it will take to store the references
341
ref_lists = <object>PyTuple_GET_ITEM_ptr_object(node, 3)
382
342
next_len = len(ref_lists) # TODO: use a Py function
384
344
# If there are no nodes, we don't need to do any work
393
353
refs_len = refs_len + (next_len - 1)
394
354
for reference in ref_list:
395
if (not PyTuple_CheckExact(reference)
396
and not StaticTuple_CheckExact(reference)):
355
if not PyTuple_CheckExact(reference):
398
357
'We expect references to be tuples not: %s'
399
358
% type(reference))
400
next_len = len(reference)
359
next_len = PyTuple_GET_SIZE(reference)
402
361
# We will need (len - 1) '\x00' characters to
403
362
# separate the reference key
404
363
refs_len = refs_len + (next_len - 1)
405
for ref_bit in reference:
406
if not PyString_CheckExact(ref_bit):
364
for i from 0 <= i < next_len:
365
ref_bit = PyTuple_GET_ITEM_ptr_object(reference, i)
366
if not PyString_CheckExact_ptr(ref_bit):
407
367
raise TypeError('We expect reference bits'
408
368
' to be strings not: %s'
409
369
% type(<object>ref_bit))
410
refs_len = refs_len + PyString_GET_SIZE(ref_bit)
370
refs_len = refs_len + PyString_GET_SIZE_ptr(ref_bit)
412
372
# So we have the (key NULL refs NULL value LF)
413
373
key_len = PyString_Size(string_key)
415
if not PyString_CheckExact(val):
374
val = PyTuple_GET_ITEM_ptr_object(node, 2)
375
if not PyString_CheckExact_ptr(val):
416
376
raise TypeError('Expected a plain str for value not: %s'
418
value = PyString_AS_STRING(val)
419
value_len = PyString_GET_SIZE(val)
378
value = PyString_AS_STRING_ptr(val)
379
value_len = PyString_GET_SIZE_ptr(val)
420
380
flat_len = (key_len + 1 + refs_len + 1 + value_len + 1)
421
381
line = PyString_FromStringAndSize(NULL, flat_len)
422
382
# Get a pointer to the new buffer
440
400
first_reference = 0
441
next_len = len(reference)
401
next_len = PyTuple_GET_SIZE(reference)
442
402
for i from 0 <= i < next_len:
446
ref_bit = reference[i]
447
ref_bit_len = PyString_GET_SIZE(ref_bit)
448
memcpy(out, PyString_AS_STRING(ref_bit), ref_bit_len)
406
ref_bit = PyTuple_GET_ITEM_ptr_object(reference, i)
407
ref_bit_len = PyString_GET_SIZE_ptr(ref_bit)
408
memcpy(out, PyString_AS_STRING_ptr(ref_bit), ref_bit_len)
449
409
out = out + ref_bit_len