1
# Copyright (C) 2009 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Definition of a class that is similar to Set with some small changes."""
19
cdef extern from "python-compat.h":
22
cdef extern from "Python.h":
23
ctypedef unsigned long size_t
24
ctypedef long (*hashfunc)(PyObject*)
25
ctypedef PyObject *(*richcmpfunc)(PyObject *, PyObject *, int)
26
ctypedef int (*visitproc)(PyObject *, void *)
27
ctypedef int (*traverseproc)(PyObject *, visitproc, void *)
30
PyObject *Py_NotImplemented
31
void Py_INCREF(PyObject *)
32
void Py_DECREF(PyObject *)
33
ctypedef struct PyTypeObject:
35
richcmpfunc tp_richcompare
36
traverseproc tp_traverse
38
PyTypeObject *Py_TYPE(PyObject *)
39
int PyObject_IsTrue(PyObject *)
41
void *PyMem_Malloc(size_t nbytes)
42
void PyMem_Free(void *)
43
void memset(void *, int, size_t)
46
# Dummy is an object used to mark nodes that have been deleted. Since
47
# collisions require us to move a node to an alternative location, if we just
48
# set an entry to NULL on delete, we won't find any relocated nodes.
49
# We have to use _dummy_obj because we need to keep a refcount to it, but we
50
# also use _dummy as a pointer, because it avoids having to put <PyObject*> all
52
cdef object _dummy_obj
55
_dummy = <PyObject *>_dummy_obj
58
cdef int _is_equal(PyObject *this, long this_hash, PyObject *other) except -1:
64
other_hash = Py_TYPE(other).tp_hash(other)
66
# Even though other successfully hashed in the past, it seems to have
67
# changed its mind, and failed this time, so propogate the failure.
69
if other_hash != this_hash:
72
# This implements a subset of the PyObject_RichCompareBool functionality.
74
# 1) Doesn't try to do anything with old-style classes
75
# 2) Assumes that both objects have a tp_richcompare implementation, and
76
# that if that is not enough to compare equal, then they are not
77
# equal. (It doesn't try to cast them both to some intermediate form
78
# that would compare equal.)
79
res = Py_TYPE(this).tp_richcompare(this, other, Py_EQ)
80
if res == NULL: # Exception
82
if PyObject_IsTrue(res):
85
if res == Py_NotImplemented:
87
res = Py_TYPE(other).tp_richcompare(other, this, Py_EQ)
90
if PyObject_IsTrue(res):
97
cdef public api class SimpleSet [object SimpleSetObject, type SimpleSet_Type]:
98
"""This class can be used to track canonical forms for objects.
100
It is similar in function to the interned dictionary that is used by
103
1) It assumes that hash(obj) is cheap, so does not need to inline a copy
105
2) It only stores one reference to the object, rather than 2 (key vs
108
As such, it uses 1/3rd the amount of memory to store a pointer to the
111
# Attributes are defined in the .pxd file
112
DEF DEFAULT_SIZE=1024
115
cdef Py_ssize_t size, n_bytes
118
self._mask = size - 1
121
n_bytes = sizeof(PyObject*) * size;
122
self._table = <PyObject **>PyMem_Malloc(n_bytes)
123
if self._table == NULL:
125
memset(self._table, 0, n_bytes)
127
def __dealloc__(self):
128
if self._table != NULL:
129
PyMem_Free(self._table)
144
def _memory_size(self):
145
"""Return the number of bytes of memory consumed by this class."""
146
return sizeof(self) + (sizeof(PyObject*)*(self._mask + 1))
151
def _test_lookup(self, key):
154
slot = _lookup(self, key)
157
elif slot[0] == _dummy:
160
res = <object>slot[0]
161
return <int>(slot - self._table), res
163
def __contains__(self, key):
164
"""Is key present in this SimpleSet."""
167
slot = _lookup(self, key)
168
if slot[0] == NULL or slot[0] == _dummy:
172
cdef PyObject *_get(self, object key) except? NULL:
173
"""Return the object (or nothing) define at the given location."""
176
slot = _lookup(self, key)
177
if slot[0] == NULL or slot[0] == _dummy:
181
def __getitem__(self, key):
182
"""Return a stored item that is equivalent to key."""
183
cdef PyObject *py_val
185
py_val = self._get(key)
187
raise KeyError("Key %s is not present" % key)
188
val = <object>(py_val)
191
cdef int _insert_clean(self, PyObject *key) except -1:
192
"""Insert a key into self.table.
194
This is only meant to be used during times like '_resize',
195
as it makes a lot of assuptions about keys not already being present,
196
and there being no dummy entries.
198
cdef size_t i, n_lookup
200
cdef PyObject **table, **slot
206
the_hash = Py_TYPE(key).tp_hash(key)
210
for n_lookup from 0 <= n_lookup <= <size_t>mask: # Don't loop forever
211
slot = &table[i & mask]
214
self._fill = self._fill + 1
215
self._used = self._used + 1
218
raise RuntimeError('ran out of slots.')
220
def _py_resize(self, min_used):
221
"""Do not use this directly, it is only exposed for testing."""
222
return self._resize(min_used)
224
cdef Py_ssize_t _resize(self, Py_ssize_t min_used) except -1:
225
"""Resize the internal table.
227
The final table will be big enough to hold at least min_used entries.
228
We will copy the data from the existing table over, leaving out dummy
231
:return: The new size of the internal table
233
cdef Py_ssize_t new_size, n_bytes, remaining
234
cdef PyObject **new_table, **old_table, **slot
236
new_size = DEFAULT_SIZE
237
while new_size <= min_used and new_size > 0:
238
new_size = new_size << 1
239
# We rolled over our signed size field
242
# Even if min_used == self._mask + 1, and we aren't changing the actual
243
# size, we will still run the algorithm so that dummy entries are
246
# if new_size < self._used:
247
# raise RuntimeError('cannot shrink SimpleSet to something'
248
# ' smaller than the number of used slots.')
249
n_bytes = sizeof(PyObject*) * new_size;
250
new_table = <PyObject **>PyMem_Malloc(n_bytes)
251
if new_table == NULL:
254
old_table = self._table
255
self._table = new_table
256
memset(self._table, 0, n_bytes)
257
self._mask = new_size - 1
259
remaining = self._fill
262
# Moving everything to the other table is refcount neutral, so we don't
266
if slot[0] == NULL: # unused slot
268
elif slot[0] == _dummy: # dummy slot
269
remaining = remaining - 1
271
remaining = remaining - 1
272
self._insert_clean(slot[0])
274
PyMem_Free(old_table)
278
"""Similar to set.add(), start tracking this key.
280
There is one small difference, which is that we return the object that
281
is stored at the given location. (which is closer to the
282
dict.setdefault() functionality.)
284
return self._add(key)
286
cdef object _add(self, key):
287
cdef PyObject **slot, *py_key
290
py_key = <PyObject *>key
291
if (Py_TYPE(py_key).tp_richcompare == NULL
292
or Py_TYPE(py_key).tp_hash == NULL):
293
raise TypeError('Types added to SimpleSet must implement'
294
' both tp_richcompare and tp_hash')
296
# We need at least one empty slot
297
assert self._used < self._mask
298
slot = _lookup(self, key)
299
if (slot[0] == NULL):
301
self._fill = self._fill + 1
302
self._used = self._used + 1
305
elif (slot[0] == _dummy):
307
self._used = self._used + 1
310
# No else: clause. If _lookup returns a pointer to
311
# a live object, then we already have a value at this location.
312
retval = <object>(slot[0])
313
# PySet and PyDict use a 2-3rds full algorithm, we'll follow suit
314
if added and (self._fill * 3) >= ((self._mask + 1) * 2):
315
# However, we always work for a load factor of 2:1
316
self._resize(self._used * 2)
317
# Even if we resized and ended up moving retval into a different slot,
318
# it is still the value that is held at the slot equivalent to 'key',
319
# so we can still return it
322
def discard(self, key):
323
"""Remove key from the set, whether it exists or not.
325
:return: False if the item did not exist, True if it did
327
if self._discard(key):
331
cdef int _discard(self, key) except -1:
332
cdef PyObject **slot, *py_key
334
slot = _lookup(self, key)
335
if slot[0] == NULL or slot[0] == _dummy:
337
self._used = self._used - 1
340
# PySet uses the heuristic: If more than 1/5 are dummies, then resize
342
# if ((so->_fill - so->_used) * 5 < so->mask)
343
# However, we are planning on using this as an interning structure, in
344
# which we will be putting a lot of objects. And we expect that large
345
# groups of them are going to have the same lifetime.
346
# Dummy entries hurt a little bit because they cause the lookup to keep
347
# searching, but resizing is also rather expensive
348
# For now, we'll just use their algorithm, but we may want to revisit
350
if ((self._fill - self._used) * 5 > self._mask):
351
self._resize(self._used * 2)
355
return _SimpleSet_iterator(self)
358
cdef class _SimpleSet_iterator:
359
"""Iterator over the SimpleSet structure."""
363
cdef Py_ssize_t _used # track if things have been mutated while iterating
364
cdef Py_ssize_t len # number of entries left
366
def __init__(self, obj):
369
self._used = self.set._used
370
self.len = self.set._used
376
cdef Py_ssize_t mask, i
381
if self.set._used != self._used:
382
# Force this exception to continue to be raised
384
raise RuntimeError("Set size changed during iteration")
385
if not SimpleSet_Next(self.set, &self.pos, &key):
389
the_key = <object>key # INCREF
390
self.len = self.len - 1
393
def __length_hint__(self):
394
if self.set is not None and self._used == self.set._used:
400
cdef api SimpleSet SimpleSet_New():
401
"""Create a new SimpleSet object."""
405
cdef SimpleSet _check_self(object self):
406
"""Check that the parameter is not None.
408
Pyrex/Cython will do type checking, but only to ensure that an object is
409
either the right type or None. You can say "object foo not None" for pure
410
python functions, but not for C functions.
411
So this is just a helper for all the apis that need to do the check.
413
cdef SimpleSet true_self
415
raise TypeError('self must not be None')
420
cdef PyObject **_lookup(SimpleSet self, object key) except NULL:
421
"""Find the slot where 'key' would fit.
423
This is the same as a dicts 'lookup' function.
425
:param key: An object we are looking up
426
:param hash: The hash for key
427
:return: The location in self.table where key should be put.
428
location == NULL is an exception, but (*location) == NULL just
429
indicates the slot is empty and can be used.
431
# This uses Quadratic Probing:
432
# http://en.wikipedia.org/wiki/Quadratic_probing
434
# This leads to probe locations at:
437
# h2 = h0 + 3 = h1 + 1 + 1
438
# h3 = h0 + 6 = h2 + 1 + 2
439
# h4 = h0 + 10 = h2 + 1 + 3
440
# Note that all of these are '& mask', but that is computed *after* the
442
# This differs from the algorithm used by Set and Dict. Which, effectively,
443
# use double-hashing, and a step size that starts large, but dwindles to
444
# stepping one-by-one.
445
# This gives more 'locality' in that if you have a collision at offset X,
446
# the first fallback is X+1, which is fast to check. However, that means
447
# that an object w/ hash X+1 will also check there, and then X+2 next.
448
# However, for objects with differing hashes, their chains are different.
449
# The former checks X, X+1, X+3, ... the latter checks X+1, X+2, X+4, ...
450
# So different hashes diverge quickly.
451
# A bigger problem is that we *only* ever use the lowest bits of the hash
452
# So all integers (x + SIZE*N) will resolve into the same bucket, and all
453
# use the same collision resolution. We may want to try to find a way to
454
# incorporate the upper bits of the hash with quadratic probing. (For
455
# example, X, X+1, X+3+some_upper_bits, X+6+more_upper_bits, etc.)
456
cdef size_t i, n_lookup
459
cdef PyObject **table, **slot, *cur, **free_slot, *py_key
461
# hash is a signed long(), we are using an offset at unsigned size_t
467
py_key = <PyObject *>key
468
for n_lookup from 0 <= n_lookup <= <size_t>mask: # Don't loop forever
469
slot = &table[i & mask]
473
if free_slot != NULL:
474
# Did we find an earlier _dummy entry?
479
# Found an exact pointer to the key
482
if free_slot == NULL:
484
elif _is_equal(py_key, key_hash, cur):
485
# Both py_key and cur belong in this slot, return it
488
raise AssertionError('should never get here')
491
cdef api PyObject **_SimpleSet_Lookup(object self, object key) except NULL:
492
"""Find the slot where 'key' would fit.
494
This is the same as a dicts 'lookup' function. This is a private
495
api because mutating what you get without maintaing the other invariants
498
:param key: An object we are looking up
499
:param hash: The hash for key
500
:return: The location in self._table where key should be put
501
should never be NULL, but may reference a NULL (PyObject*)
503
return _lookup(_check_self(self), key)
506
cdef api object SimpleSet_Add(object self, object key):
507
"""Add a key to the SimpleSet (set).
509
:param self: The SimpleSet to add the key to.
510
:param key: The key to be added. If the key is already present,
511
self will not be modified
512
:return: The current key stored at the location defined by 'key'.
513
This may be the same object, or it may be an equivalent object.
514
(consider dict.setdefault(key, key))
516
return _check_self(self)._add(key)
519
cdef api int SimpleSet_Contains(object self, object key) except -1:
520
"""Is key present in self?"""
521
return (key in _check_self(self))
524
cdef api int SimpleSet_Discard(object self, object key) except -1:
525
"""Remove the object referenced at location 'key'.
527
:param self: The SimpleSet being modified
528
:param key: The key we are checking on
529
:return: 1 if there was an object present, 0 if there was not, and -1 on
532
return _check_self(self)._discard(key)
535
cdef api PyObject *SimpleSet_Get(SimpleSet self, object key) except? NULL:
536
"""Get a pointer to the object present at location 'key'.
538
This returns an object which is equal to key which was previously added to
539
self. This returns a borrowed reference, as it may also return NULL if no
540
value is present at that location.
542
:param key: The value we are looking for
543
:return: The object present at that location
545
return _check_self(self)._get(key)
548
cdef api Py_ssize_t SimpleSet_Size(object self) except -1:
549
"""Get the number of active entries in 'self'"""
550
return _check_self(self)._used
553
cdef api int SimpleSet_Next(object self, Py_ssize_t *pos, PyObject **key):
554
"""Walk over items in a SimpleSet.
556
:param pos: should be initialized to 0 by the caller, and will be updated
558
:param key: Will return a borrowed reference to key
559
:return: 0 if nothing left, 1 if we are returning a new value
561
cdef Py_ssize_t i, mask
562
cdef SimpleSet true_self
563
cdef PyObject **table
564
true_self = _check_self(self)
568
mask = true_self._mask
569
table= true_self._table
570
while (i <= mask and (table[i] == NULL or table[i] == _dummy)):
580
cdef int SimpleSet_traverse(SimpleSet self, visitproc visit, void *arg):
581
"""This is an implementation of 'tp_traverse' that hits the whole table.
583
Cython/Pyrex don't seem to let you define a tp_traverse, and they only
584
define one for you if you have an 'object' attribute. Since they don't
585
support C arrays of objects, we access the PyObject * directly.
588
cdef PyObject *next_key
592
while SimpleSet_Next(self, &pos, &next_key):
593
ret = visit(next_key, arg)
598
# It is a little bit ugly to do this, but it works, and means that Meliae can
599
# dump the total memory consumed by all child objects.
600
(<PyTypeObject *>SimpleSet).tp_traverse = <traverseproc>SimpleSet_traverse
added
removed
bzrlib/_simple_set_pyx.pyx
