bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
1 |
# Copyright (C) 2008 Canonical Ltd
|
2 |
#
|
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
||
17 |
"""Tests for maps built on a CHK versionedfiles facility."""
|
|
18 |
||
|
3735.9.1
by John Arbash Meinel
Start working on an iter_interesting_nodes, which can find nodes to transmit |
19 |
from itertools import izip |
20 |
||
21 |
from bzrlib import chk_map |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
22 |
from bzrlib.chk_map import ( |
23 |
CHKMap, |
|
24 |
InternalNode, |
|
25 |
LeafNode, |
|
26 |
_deserialise, |
|
27 |
)
|
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
28 |
from bzrlib.tests import TestCaseWithTransport |
29 |
||
30 |
||
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
31 |
class TestCaseWithStore(TestCaseWithTransport): |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
32 |
|
33 |
def get_chk_bytes(self): |
|
34 |
# The eassiest way to get a CHK store is a development3 repository and
|
|
35 |
# then work with the chk_bytes attribute directly.
|
|
36 |
repo = self.make_repository(".", format="development3") |
|
37 |
repo.lock_write() |
|
38 |
self.addCleanup(repo.unlock) |
|
39 |
repo.start_write_group() |
|
40 |
self.addCleanup(repo.abort_write_group) |
|
41 |
return repo.chk_bytes |
|
42 |
||
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
43 |
def _get_map(self, a_dict, maximum_size=0, chk_bytes=None): |
44 |
if chk_bytes is None: |
|
45 |
chk_bytes = self.get_chk_bytes() |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
46 |
root_key = CHKMap.from_dict(chk_bytes, a_dict, maximum_size=maximum_size) |
47 |
chkmap = CHKMap(chk_bytes, root_key) |
|
48 |
return chkmap |
|
49 |
||
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
50 |
def read_bytes(self, chk_bytes, key): |
51 |
stream = chk_bytes.get_record_stream([key], 'unordered', True) |
|
52 |
return stream.next().get_bytes_as("fulltext") |
|
53 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
54 |
def to_dict(self, node, *args): |
55 |
return dict(node.iteritems(*args)) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
56 |
|
57 |
||
58 |
class TestMap(TestCaseWithStore): |
|
59 |
||
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
60 |
def assertHasABMap(self, chk_bytes): |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
61 |
root_key = ('sha1:f14dd34def95036bc06bb5c0ed95437d7383a04a',) |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
62 |
self.assertEqual( |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
63 |
'chkleaf:\n0\n1\n1\na\x00b\n', |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
64 |
self.read_bytes(chk_bytes, root_key)) |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
65 |
return root_key |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
66 |
|
67 |
def assertHasEmptyMap(self, chk_bytes): |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
68 |
root_key = ('sha1:4e6482a3a5cb2d61699971ac77befe11a0ec5779',) |
69 |
self.assertEqual("chkleaf:\n0\n1\n0\n", self.read_bytes(chk_bytes, root_key)) |
|
70 |
return root_key |
|
|
3735.2.17
by Robert Collins
Cache node length to avoid full iteration on __len__ calls. |
71 |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
72 |
def test_from_dict_empty(self): |
73 |
chk_bytes = self.get_chk_bytes() |
|
74 |
root_key = CHKMap.from_dict(chk_bytes, {}) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
75 |
# Check the data was saved and inserted correctly.
|
76 |
expected_root_key = self.assertHasEmptyMap(chk_bytes) |
|
77 |
self.assertEqual(expected_root_key, root_key) |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
78 |
|
79 |
def test_from_dict_ab(self): |
|
80 |
chk_bytes = self.get_chk_bytes() |
|
81 |
root_key = CHKMap.from_dict(chk_bytes, {"a":"b"}) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
82 |
# Check the data was saved and inserted correctly.
|
83 |
expected_root_key = self.assertHasABMap(chk_bytes) |
|
84 |
self.assertEqual(expected_root_key, root_key) |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
85 |
|
86 |
def test_apply_empty_ab(self): |
|
87 |
# applying a delta (None, "a", "b") to an empty chkmap generates the
|
|
88 |
# same map as from_dict_ab.
|
|
89 |
chk_bytes = self.get_chk_bytes() |
|
90 |
root_key = CHKMap.from_dict(chk_bytes, {}) |
|
91 |
chkmap = CHKMap(chk_bytes, root_key) |
|
92 |
new_root = chkmap.apply_delta([(None, "a", "b")]) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
93 |
# Check the data was saved and inserted correctly.
|
94 |
expected_root_key = self.assertHasABMap(chk_bytes) |
|
95 |
self.assertEqual(expected_root_key, new_root) |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
96 |
# The update should have left us with an in memory root node, with an
|
97 |
# updated key.
|
|
98 |
self.assertEqual(new_root, chkmap._root_node._key) |
|
99 |
||
100 |
def test_apply_ab_empty(self): |
|
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
101 |
# applying a delta ("a", None, None) to a map with 'a' in it generates
|
102 |
# an empty map.
|
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
103 |
chk_bytes = self.get_chk_bytes() |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
104 |
root_key = CHKMap.from_dict(chk_bytes, {("a",):"b"}) |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
105 |
chkmap = CHKMap(chk_bytes, root_key) |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
106 |
new_root = chkmap.apply_delta([(("a",), None, None)]) |
107 |
# Check the data was saved and inserted correctly.
|
|
108 |
expected_root_key = self.assertHasEmptyMap(chk_bytes) |
|
109 |
self.assertEqual(expected_root_key, new_root) |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
110 |
# The update should have left us with an in memory root node, with an
|
111 |
# updated key.
|
|
112 |
self.assertEqual(new_root, chkmap._root_node._key) |
|
113 |
||
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
114 |
def test_iter_changes_empty_ab(self): |
115 |
# Asking for changes between an empty dict to a dict with keys returns
|
|
116 |
# all the keys.
|
|
|
3735.2.31
by Robert Collins
CHKMap.iter_changes |
117 |
basis = self._get_map({}, maximum_size=10) |
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
118 |
target = self._get_map( |
119 |
{('a',): 'content here', ('b',): 'more content'}, |
|
|
3735.2.31
by Robert Collins
CHKMap.iter_changes |
120 |
chk_bytes=basis._store, maximum_size=10) |
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
121 |
self.assertEqual([(('a',), None, 'content here'), |
|
3735.2.31
by Robert Collins
CHKMap.iter_changes |
122 |
(('b',), None, 'more content')], |
123 |
sorted(list(target.iter_changes(basis)))) |
|
124 |
||
125 |
def test_iter_changes_ab_empty(self): |
|
126 |
# Asking for changes between a dict with keys to an empty dict returns
|
|
127 |
# all the keys.
|
|
128 |
basis = self._get_map({('a',): 'content here', ('b',): 'more content'}, |
|
129 |
maximum_size=10) |
|
130 |
target = self._get_map({}, chk_bytes=basis._store, maximum_size=10) |
|
131 |
self.assertEqual([(('a',), 'content here', None), |
|
132 |
(('b',), 'more content', None)], |
|
133 |
sorted(list(target.iter_changes(basis)))) |
|
134 |
||
135 |
def test_iter_changes_empty_empty_is_empty(self): |
|
136 |
basis = self._get_map({}, maximum_size=10) |
|
137 |
target = self._get_map({}, chk_bytes=basis._store, maximum_size=10) |
|
138 |
self.assertEqual([], sorted(list(target.iter_changes(basis)))) |
|
139 |
||
140 |
def test_iter_changes_ab_ab_is_empty(self): |
|
141 |
basis = self._get_map({('a',): 'content here', ('b',): 'more content'}, |
|
142 |
maximum_size=10) |
|
143 |
target = self._get_map( |
|
144 |
{('a',): 'content here', ('b',): 'more content'}, |
|
145 |
chk_bytes=basis._store, maximum_size=10) |
|
146 |
self.assertEqual([], sorted(list(target.iter_changes(basis)))) |
|
147 |
||
148 |
def test_iter_changes_ab_ab_nodes_not_loaded(self): |
|
149 |
basis = self._get_map({('a',): 'content here', ('b',): 'more content'}, |
|
150 |
maximum_size=10) |
|
151 |
target = self._get_map( |
|
152 |
{('a',): 'content here', ('b',): 'more content'}, |
|
153 |
chk_bytes=basis._store, maximum_size=10) |
|
154 |
list(target.iter_changes(basis)) |
|
155 |
self.assertIsInstance(target._root_node, tuple) |
|
156 |
self.assertIsInstance(basis._root_node, tuple) |
|
157 |
||
158 |
def test_iter_changes_ab_ab_changed_values_shown(self): |
|
159 |
basis = self._get_map({('a',): 'content here', ('b',): 'more content'}, |
|
160 |
maximum_size=10) |
|
161 |
target = self._get_map( |
|
162 |
{('a',): 'content here', ('b',): 'different content'}, |
|
163 |
chk_bytes=basis._store, maximum_size=10) |
|
164 |
result = sorted(list(target.iter_changes(basis))) |
|
165 |
self.assertEqual([(('b',), 'more content', 'different content')], |
|
166 |
result) |
|
167 |
||
168 |
def test_iter_changes_mixed_node_length(self): |
|
169 |
# When one side has different node lengths than the other, common
|
|
170 |
# but different keys still need to be show, and new-and-old included
|
|
171 |
# appropriately.
|
|
172 |
# aaa - common unaltered
|
|
173 |
# aab - common altered
|
|
174 |
# b - basis only
|
|
175 |
# at - target only
|
|
176 |
# we expect:
|
|
177 |
# aaa to be not loaded (later test)
|
|
178 |
# aab, b, at to be returned.
|
|
179 |
# basis splits at byte 0,1,2, aaa is commonb is basis only
|
|
180 |
basis_dict = {('aaa',): 'foo bar', |
|
181 |
('aab',): 'common altered a', ('b',): 'foo bar b'} |
|
182 |
# target splits at byte 1,2, at is target only
|
|
183 |
target_dict = {('aaa',): 'foo bar', |
|
184 |
('aab',): 'common altered b', ('at',): 'foo bar t'} |
|
185 |
changes = [ |
|
186 |
(('aab',), 'common altered a', 'common altered b'), |
|
187 |
(('at',), None, 'foo bar t'), |
|
188 |
(('b',), 'foo bar b', None), |
|
189 |
]
|
|
190 |
basis = self._get_map(basis_dict, maximum_size=10) |
|
191 |
target = self._get_map(target_dict, maximum_size=10, |
|
192 |
chk_bytes=basis._store) |
|
193 |
self.assertEqual(changes, sorted(list(target.iter_changes(basis)))) |
|
194 |
||
195 |
def test_iter_changes_common_pages_not_loaded(self): |
|
196 |
# aaa - common unaltered
|
|
197 |
# aab - common altered
|
|
198 |
# b - basis only
|
|
199 |
# at - target only
|
|
200 |
# we expect:
|
|
201 |
# aaa to be not loaded
|
|
202 |
# aaa not to be in result.
|
|
203 |
basis_dict = {('aaa',): 'foo bar', |
|
204 |
('aab',): 'common altered a', ('b',): 'foo bar b'} |
|
205 |
# target splits at byte 1, at is target only
|
|
206 |
target_dict = {('aaa',): 'foo bar', |
|
207 |
('aab',): 'common altered b', ('at',): 'foo bar t'} |
|
208 |
basis = self._get_map(basis_dict, maximum_size=10) |
|
209 |
target = self._get_map(target_dict, maximum_size=10, |
|
210 |
chk_bytes=basis._store) |
|
|
3735.2.32
by Robert Collins
Activate test for common node skipping. - 50 times performance improvement. |
211 |
basis_get = basis._store.get_record_stream |
212 |
def get_record_stream(keys, order, fulltext): |
|
213 |
if ('sha1:1adf7c0d1b9140ab5f33bb64c6275fa78b1580b7',) in keys: |
|
214 |
self.fail("'aaa' pointer was followed %r" % keys) |
|
215 |
return basis_get(keys, order, fulltext) |
|
216 |
basis._store.get_record_stream = get_record_stream |
|
|
3735.2.31
by Robert Collins
CHKMap.iter_changes |
217 |
result = sorted(list(target.iter_changes(basis))) |
218 |
for change in result: |
|
219 |
if change[0] == ('aaa',): |
|
220 |
self.fail("Found unexpected change: %s" % change) |
|
221 |
||
222 |
def test_iter_changes_unchanged_keys_in_multi_key_leafs_ignored(self): |
|
223 |
# Within a leaf there are no hash's to exclude keys, make sure multi
|
|
224 |
# value leaf nodes are handled well.
|
|
225 |
basis_dict = {('aaa',): 'foo bar', |
|
226 |
('aab',): 'common altered a', ('b',): 'foo bar b'} |
|
227 |
target_dict = {('aaa',): 'foo bar', |
|
228 |
('aab',): 'common altered b', ('at',): 'foo bar t'} |
|
229 |
changes = [ |
|
230 |
(('aab',), 'common altered a', 'common altered b'), |
|
231 |
(('at',), None, 'foo bar t'), |
|
232 |
(('b',), 'foo bar b', None), |
|
233 |
]
|
|
234 |
basis = self._get_map(basis_dict) |
|
235 |
target = self._get_map(target_dict, chk_bytes=basis._store) |
|
236 |
self.assertEqual(changes, sorted(list(target.iter_changes(basis)))) |
|
|
3735.2.30
by Robert Collins
Start iter_changes between CHKMap instances. |
237 |
|
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
238 |
def test_iteritems_empty(self): |
239 |
chk_bytes = self.get_chk_bytes() |
|
240 |
root_key = CHKMap.from_dict(chk_bytes, {}) |
|
241 |
chkmap = CHKMap(chk_bytes, root_key) |
|
242 |
self.assertEqual([], list(chkmap.iteritems())) |
|
243 |
||
244 |
def test_iteritems_two_items(self): |
|
245 |
chk_bytes = self.get_chk_bytes() |
|
246 |
root_key = CHKMap.from_dict(chk_bytes, |
|
247 |
{"a":"content here", "b":"more content"}) |
|
248 |
chkmap = CHKMap(chk_bytes, root_key) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
249 |
self.assertEqual([(("a",), "content here"), (("b",), "more content")], |
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
250 |
sorted(list(chkmap.iteritems()))) |
251 |
||
|
3735.2.9
by Robert Collins
Get a working chk_map using inventory implementation bootstrapped. |
252 |
def test_iteritems_selected_one_of_two_items(self): |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
253 |
chkmap = self._get_map( {("a",):"content here", ("b",):"more content"}) |
254 |
self.assertEqual({("a",): "content here"}, |
|
255 |
self.to_dict(chkmap, [("a",)])) |
|
|
3735.2.9
by Robert Collins
Get a working chk_map using inventory implementation bootstrapped. |
256 |
|
|
3735.2.17
by Robert Collins
Cache node length to avoid full iteration on __len__ calls. |
257 |
def test___len__empty(self): |
258 |
chkmap = self._get_map({}) |
|
259 |
self.assertEqual(0, len(chkmap)) |
|
260 |
||
261 |
def test___len__2(self): |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
262 |
chkmap = self._get_map({"foo":"bar", "gam":"quux"}) |
263 |
self.assertEqual(2, len(chkmap)) |
|
264 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
265 |
def test_max_size_100_bytes_new(self): |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
266 |
# When there is a 100 byte upper node limit, a tree is formed.
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
267 |
chkmap = self._get_map({("k1"*50,):"v1", ("k2"*50,):"v2"}, maximum_size=100) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
268 |
# We expect three nodes:
|
269 |
# A root, with two children, and with two key prefixes - k1 to one, and
|
|
270 |
# k2 to the other as our node splitting is only just being developed.
|
|
271 |
# The maximum size should be embedded
|
|
272 |
chkmap._ensure_root() |
|
273 |
self.assertEqual(100, chkmap._root_node.maximum_size) |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
274 |
self.assertEqual(1, chkmap._root_node._key_width) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
275 |
# There should be two child nodes, and prefix of 2(bytes):
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
276 |
self.assertEqual(2, len(chkmap._root_node._items)) |
277 |
self.assertEqual("k", chkmap._root_node.unique_serialised_prefix()) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
278 |
# The actual nodes pointed at will change as serialisers change; so
|
279 |
# here we test that the key prefix is correct; then load the nodes and
|
|
280 |
# check they have the right pointed at key; whether they have the
|
|
281 |
# pointed at value inline or not is also unrelated to this test so we
|
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
282 |
# don't check that in detail - rather we just check the aggregate
|
283 |
# value.
|
|
284 |
nodes = sorted(chkmap._root_node._items.items()) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
285 |
ptr1 = nodes[0] |
286 |
ptr2 = nodes[1] |
|
287 |
self.assertEqual('k1', ptr1[0]) |
|
288 |
self.assertEqual('k2', ptr2[0]) |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
289 |
node1 = _deserialise(chkmap._read_bytes(ptr1[1]), ptr1[1]) |
290 |
self.assertIsInstance(node1, LeafNode) |
|
291 |
self.assertEqual(1, len(node1)) |
|
292 |
self.assertEqual({('k1'*50,): 'v1'}, self.to_dict(node1, chkmap._store)) |
|
293 |
node2 = _deserialise(chkmap._read_bytes(ptr2[1]), ptr2[1]) |
|
294 |
self.assertIsInstance(node2, LeafNode) |
|
295 |
self.assertEqual(1, len(node2)) |
|
296 |
self.assertEqual({('k2'*50,): 'v2'}, self.to_dict(node2, chkmap._store)) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
297 |
# Having checked we have a good structure, check that the content is
|
298 |
# still accessible.
|
|
299 |
self.assertEqual(2, len(chkmap)) |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
300 |
self.assertEqual({("k1"*50,): "v1", ("k2"*50,): "v2"}, |
301 |
self.to_dict(chkmap)) |
|
302 |
||
303 |
def test_init_root_is_LeafNode_new(self): |
|
304 |
chk_bytes = self.get_chk_bytes() |
|
305 |
chkmap = CHKMap(chk_bytes, None) |
|
306 |
self.assertIsInstance(chkmap._root_node, LeafNode) |
|
307 |
self.assertEqual({}, self.to_dict(chkmap)) |
|
308 |
self.assertEqual(0, len(chkmap)) |
|
309 |
||
310 |
def test_init_and_save_new(self): |
|
311 |
chk_bytes = self.get_chk_bytes() |
|
312 |
chkmap = CHKMap(chk_bytes, None) |
|
313 |
key = chkmap._save() |
|
314 |
leaf_node = LeafNode() |
|
315 |
self.assertEqual([key], leaf_node.serialise(chk_bytes)) |
|
316 |
||
317 |
def test_map_first_item_new(self): |
|
318 |
chk_bytes = self.get_chk_bytes() |
|
319 |
chkmap = CHKMap(chk_bytes, None) |
|
320 |
chkmap.map(("foo,",), "bar") |
|
321 |
self.assertEqual({('foo,',): 'bar'}, self.to_dict(chkmap)) |
|
322 |
self.assertEqual(1, len(chkmap)) |
|
323 |
key = chkmap._save() |
|
324 |
leaf_node = LeafNode() |
|
325 |
leaf_node.map(chk_bytes, ("foo,",), "bar") |
|
326 |
self.assertEqual([key], leaf_node.serialise(chk_bytes)) |
|
327 |
||
328 |
def test_unmap_last_item_root_is_leaf_new(self): |
|
329 |
chkmap = self._get_map({("k1"*50,): "v1", ("k2"*50,): "v2"}) |
|
330 |
chkmap.unmap(("k1"*50,)) |
|
331 |
chkmap.unmap(("k2"*50,)) |
|
332 |
self.assertEqual(0, len(chkmap)) |
|
333 |
self.assertEqual({}, self.to_dict(chkmap)) |
|
334 |
key = chkmap._save() |
|
335 |
leaf_node = LeafNode() |
|
336 |
self.assertEqual([key], leaf_node.serialise(chkmap._store)) |
|
|
3735.2.17
by Robert Collins
Cache node length to avoid full iteration on __len__ calls. |
337 |
|
|
3735.9.2
by John Arbash Meinel
Add a _dump_tree helper that assists in debugging what is going on. |
338 |
def test__dump_tree(self): |
339 |
chkmap = self._get_map({("aaa",): "value1", ("aab",): "value2", |
|
340 |
("bbb",): "value3",}, |
|
341 |
maximum_size=10) |
|
342 |
self.assertEqualDiff( |
|
343 |
"'' InternalNode sha1:cd9b68f18c9754a79065b06379fba543f9031742\n" |
|
344 |
" 'a' InternalNode sha1:ed0ceb5aeb87c56df007a17997134328ff4d0b8d\n" |
|
345 |
" 'aaa' LeafNode sha1:16fa5a38b80d29b529afc45f7a4f894650fc067f\n" |
|
346 |
" ('aaa',) 'value1'\n" |
|
347 |
" 'aab' LeafNode sha1:8fca5400dc99ef1b464e60ca25da53b57406ed38\n" |
|
348 |
" ('aab',) 'value2'\n" |
|
349 |
" 'b' LeafNode sha1:67f15d1dfa451d388ed08ff17b4f9578ba010d01\n" |
|
350 |
" ('bbb',) 'value3'\n", |
|
351 |
chkmap._dump_tree()) |
|
352 |
||
|
3735.2.8
by Robert Collins
New chk_map module for use in tree based inventory storage. |
353 |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
354 |
class TestLeafNode(TestCaseWithStore): |
355 |
||
356 |
def test_current_size_empty(self): |
|
357 |
node = LeafNode() |
|
358 |
self.assertEqual(15, node._current_size()) |
|
359 |
||
360 |
def test_current_size_size_changed(self): |
|
361 |
node = LeafNode() |
|
362 |
node.set_maximum_size(10) |
|
363 |
self.assertEqual(16, node._current_size()) |
|
364 |
||
365 |
def test_current_size_width_changed(self): |
|
366 |
node = LeafNode() |
|
367 |
node._key_width = 10 |
|
368 |
self.assertEqual(16, node._current_size()) |
|
369 |
||
370 |
def test_current_size_items(self): |
|
371 |
node = LeafNode() |
|
372 |
base_size = node._current_size() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
373 |
node.map(None, ("foo bar",), "baz") |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
374 |
self.assertEqual(base_size + 12, node._current_size()) |
375 |
||
376 |
def test_deserialise_empty(self): |
|
377 |
node = LeafNode.deserialise("chkleaf:\n10\n1\n0\n", ("sha1:1234",)) |
|
378 |
self.assertEqual(0, len(node)) |
|
379 |
self.assertEqual(10, node.maximum_size) |
|
380 |
self.assertEqual(("sha1:1234",), node.key()) |
|
381 |
||
382 |
def test_deserialise_items(self): |
|
383 |
node = LeafNode.deserialise( |
|
384 |
"chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",)) |
|
385 |
self.assertEqual(2, len(node)) |
|
386 |
self.assertEqual([(("foo bar",), "baz"), (("quux",), "blarh")], |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
387 |
sorted(node.iteritems(None))) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
388 |
|
|
3735.2.25
by Robert Collins
CHKInventory core tests passing. |
389 |
def test_deserialise_item_with_null_width_1(self): |
390 |
node = LeafNode.deserialise( |
|
391 |
"chkleaf:\n0\n1\n2\nfoo\x00bar\x00baz\nquux\x00blarh\n", |
|
392 |
("sha1:1234",)) |
|
393 |
self.assertEqual(2, len(node)) |
|
394 |
self.assertEqual([(("foo",), "bar\x00baz"), (("quux",), "blarh")], |
|
395 |
sorted(node.iteritems(None))) |
|
396 |
||
397 |
def test_deserialise_item_with_null_width_2(self): |
|
398 |
node = LeafNode.deserialise( |
|
399 |
"chkleaf:\n0\n2\n2\nfoo\x001\x00bar\x00baz\nquux\x00\x00blarh\n", |
|
400 |
("sha1:1234",)) |
|
401 |
self.assertEqual(2, len(node)) |
|
402 |
self.assertEqual([(("foo", "1"), "bar\x00baz"), (("quux", ""), "blarh")], |
|
403 |
sorted(node.iteritems(None))) |
|
404 |
||
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
405 |
def test_iteritems_selected_one_of_two_items(self): |
406 |
node = LeafNode.deserialise( |
|
407 |
"chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",)) |
|
408 |
self.assertEqual(2, len(node)) |
|
409 |
self.assertEqual([(("quux",), "blarh")], |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
410 |
sorted(node.iteritems(None, [("quux",), ("qaz",)]))) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
411 |
|
412 |
def test_key_new(self): |
|
413 |
node = LeafNode() |
|
414 |
self.assertEqual(None, node.key()) |
|
415 |
||
416 |
def test_key_after_map(self): |
|
417 |
node = LeafNode.deserialise("chkleaf:\n10\n1\n0\n", ("sha1:1234",)) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
418 |
node.map(None, ("foo bar",), "baz quux") |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
419 |
self.assertEqual(None, node.key()) |
420 |
||
421 |
def test_key_after_unmap(self): |
|
422 |
node = LeafNode.deserialise( |
|
423 |
"chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",)) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
424 |
node.unmap(None, ("foo bar",)) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
425 |
self.assertEqual(None, node.key()) |
426 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
427 |
def test_map_exceeding_max_size_only_entry_new(self): |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
428 |
node = LeafNode() |
429 |
node.set_maximum_size(10) |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
430 |
result = node.map(None, ("foo bar",), "baz quux") |
431 |
self.assertEqual(("foo bar", [("", node)]), result) |
|
432 |
self.assertTrue(10 < node._current_size()) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
433 |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
434 |
def test_map_exceeding_max_size_second_entry_early_difference_new(self): |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
435 |
node = LeafNode() |
436 |
node.set_maximum_size(10) |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
437 |
node.map(None, ("foo bar",), "baz quux") |
438 |
prefix, result = list(node.map(None, ("blue",), "red")) |
|
439 |
self.assertEqual("", prefix) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
440 |
self.assertEqual(2, len(result)) |
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
441 |
split_chars = set([result[0][0], result[1][0]]) |
442 |
self.assertEqual(set(["f", "b"]), split_chars) |
|
443 |
nodes = dict(result) |
|
444 |
node = nodes["f"] |
|
445 |
self.assertEqual({("foo bar",): "baz quux"}, self.to_dict(node, None)) |
|
446 |
self.assertEqual(10, node.maximum_size) |
|
447 |
self.assertEqual(1, node._key_width) |
|
448 |
node = nodes["b"] |
|
449 |
self.assertEqual({("blue",): "red"}, self.to_dict(node, None)) |
|
450 |
self.assertEqual(10, node.maximum_size) |
|
451 |
self.assertEqual(1, node._key_width) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
452 |
|
453 |
def test_map_first(self): |
|
454 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
455 |
result = node.map(None, ("foo bar",), "baz quux") |
456 |
self.assertEqual(("foo bar", [("", node)]), result) |
|
457 |
self.assertEqual({("foo bar",):"baz quux"}, self.to_dict(node, None)) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
458 |
self.assertEqual(1, len(node)) |
459 |
||
460 |
def test_map_second(self): |
|
461 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
462 |
node.map(None, ("foo bar",), "baz quux") |
463 |
result = node.map(None, ("bingo",), "bango") |
|
464 |
self.assertEqual(("", [("", node)]), result) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
465 |
self.assertEqual({("foo bar",):"baz quux", ("bingo",):"bango"}, |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
466 |
self.to_dict(node, None)) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
467 |
self.assertEqual(2, len(node)) |
468 |
||
469 |
def test_map_replacement(self): |
|
470 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
471 |
node.map(None, ("foo bar",), "baz quux") |
472 |
result = node.map(None, ("foo bar",), "bango") |
|
473 |
self.assertEqual(("foo bar", [("", node)]), result) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
474 |
self.assertEqual({("foo bar",): "bango"}, |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
475 |
self.to_dict(node, None)) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
476 |
self.assertEqual(1, len(node)) |
477 |
||
478 |
def test_serialise_empty(self): |
|
479 |
store = self.get_chk_bytes() |
|
480 |
node = LeafNode() |
|
481 |
node.set_maximum_size(10) |
|
482 |
expected_key = ("sha1:62cc3565b48b0e830216e652cf99c6bd6b05b4b9",) |
|
483 |
self.assertEqual([expected_key], |
|
484 |
list(node.serialise(store))) |
|
485 |
self.assertEqual("chkleaf:\n10\n1\n0\n", self.read_bytes(store, expected_key)) |
|
486 |
self.assertEqual(expected_key, node.key()) |
|
487 |
||
488 |
def test_serialise_items(self): |
|
489 |
store = self.get_chk_bytes() |
|
490 |
node = LeafNode() |
|
491 |
node.set_maximum_size(10) |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
492 |
node.map(None, ("foo bar",), "baz quux") |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
493 |
expected_key = ("sha1:d44cb6f0299b7e047da7f9e98f810e98f1dce1a7",) |
494 |
self.assertEqual([expected_key], |
|
495 |
list(node.serialise(store))) |
|
496 |
self.assertEqual("chkleaf:\n10\n1\n1\nfoo bar\x00baz quux\n", |
|
497 |
self.read_bytes(store, expected_key)) |
|
498 |
self.assertEqual(expected_key, node.key()) |
|
499 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
500 |
def test_unique_serialised_prefix_empty_new(self): |
501 |
node = LeafNode() |
|
502 |
self.assertEqual("", node.unique_serialised_prefix()) |
|
503 |
||
504 |
def test_unique_serialised_prefix_one_item_new(self): |
|
505 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
506 |
node.map(None, ("foo bar", "baz"), "baz quux") |
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
507 |
self.assertEqual("foo bar\x00baz", node.unique_serialised_prefix()) |
508 |
||
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
509 |
def test_unmap_missing(self): |
510 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
511 |
self.assertRaises(KeyError, node.unmap, None, ("foo bar",)) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
512 |
|
513 |
def test_unmap_present(self): |
|
514 |
node = LeafNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
515 |
node.map(None, ("foo bar",), "baz quux") |
516 |
result = node.unmap(None, ("foo bar",)) |
|
517 |
self.assertEqual(node, result) |
|
518 |
self.assertEqual({}, self.to_dict(node, None)) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
519 |
self.assertEqual(0, len(node)) |
520 |
||
521 |
||
522 |
class TestInternalNode(TestCaseWithStore): |
|
523 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
524 |
def test_add_node_empty_new(self): |
525 |
node = InternalNode() |
|
526 |
child = LeafNode() |
|
527 |
child.set_maximum_size(100) |
|
528 |
child.map(None, ("foo",), "bar") |
|
529 |
node.add_node("foo", child) |
|
530 |
# Note that node isn't strictly valid now as a tree (only one child),
|
|
531 |
# but thats ok for this test.
|
|
532 |
# The first child defines the node's width:
|
|
533 |
self.assertEqual(3, node._node_width) |
|
534 |
# We should be able to iterate over the contents without doing IO.
|
|
535 |
self.assertEqual({('foo',): 'bar'}, self.to_dict(node, None)) |
|
536 |
# The length should be known:
|
|
537 |
self.assertEqual(1, len(node)) |
|
538 |
# serialising the node should serialise the child and the node.
|
|
539 |
chk_bytes = self.get_chk_bytes() |
|
540 |
keys = list(node.serialise(chk_bytes)) |
|
541 |
child_key = child.serialise(chk_bytes)[0] |
|
542 |
self.assertEqual( |
|
543 |
[child_key, ('sha1:db23b260c2bf46bf7446c39f91668900a2491610',)], |
|
544 |
keys) |
|
545 |
# We should be able to access deserialised content.
|
|
546 |
bytes = self.read_bytes(chk_bytes, keys[1]) |
|
547 |
node = _deserialise(bytes, keys[1]) |
|
548 |
self.assertEqual(1, len(node)) |
|
549 |
self.assertEqual({('foo',): 'bar'}, self.to_dict(node, chk_bytes)) |
|
550 |
self.assertEqual(3, node._node_width) |
|
551 |
||
552 |
def test_add_node_resets_key_new(self): |
|
553 |
node = InternalNode() |
|
554 |
child = LeafNode() |
|
555 |
child.set_maximum_size(100) |
|
556 |
child.map(None, ("foo",), "bar") |
|
557 |
node.add_node("foo", child) |
|
558 |
chk_bytes = self.get_chk_bytes() |
|
559 |
keys = list(node.serialise(chk_bytes)) |
|
560 |
self.assertEqual(keys[1], node._key) |
|
561 |
node.add_node("fos", child) |
|
562 |
self.assertEqual(None, node._key) |
|
563 |
||
564 |
# def test_add_node_empty_oversized_one_ok_new(self):
|
|
565 |
# def test_add_node_one_oversized_second_kept_minimum_fan(self):
|
|
566 |
# def test_add_node_two_oversized_third_kept_minimum_fan(self):
|
|
567 |
# def test_add_node_one_oversized_second_splits_errors(self):
|
|
568 |
||
569 |
def test_iteritems_empty_new(self): |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
570 |
node = InternalNode() |
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
571 |
self.assertEqual([], sorted(node.iteritems(None))) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
572 |
|
573 |
def test_iteritems_two_children(self): |
|
574 |
node = InternalNode() |
|
575 |
leaf1 = LeafNode() |
|
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
576 |
leaf1.map(None, ('foo bar',), 'quux') |
577 |
leaf2 = LeafNode() |
|
578 |
leaf2.map(None, ('strange',), 'beast') |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
579 |
node.add_node("f", leaf1) |
580 |
node.add_node("s", leaf2) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
581 |
self.assertEqual([(('foo bar',), 'quux'), (('strange',), 'beast')], |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
582 |
sorted(node.iteritems(None))) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
583 |
|
584 |
def test_iteritems_two_children_partial(self): |
|
585 |
node = InternalNode() |
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
586 |
leaf1 = LeafNode() |
587 |
leaf1.map(None, ('foo bar',), 'quux') |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
588 |
leaf2 = LeafNode() |
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
589 |
leaf2.map(None, ('strange',), 'beast') |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
590 |
node.add_node("f", leaf1) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
591 |
# This sets up a path that should not be followed - it will error if
|
592 |
# the code tries to.
|
|
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
593 |
node._items['f'] = None |
594 |
node.add_node("s", leaf2) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
595 |
self.assertEqual([(('strange',), 'beast')], |
|
3735.2.24
by Robert Collins
test_chk_map tests all passing. |
596 |
sorted(node.iteritems(None, [('strange',), ('weird',)]))) |
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
597 |
|
598 |
def test_iteritems_partial_empty(self): |
|
599 |
node = InternalNode() |
|
600 |
self.assertEqual([], sorted(node.iteritems([('missing',)]))) |
|
601 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
602 |
def test_map_to_new_child_new(self): |
603 |
chkmap = self._get_map({('k1',):'foo', ('k2',):'bar'}, maximum_size=10) |
|
604 |
chkmap._ensure_root() |
|
605 |
node = chkmap._root_node |
|
606 |
# Ensure test validity: nothing paged in below the root.
|
|
607 |
self.assertEqual(2, |
|
608 |
len([value for value in node._items.values() |
|
609 |
if type(value) == tuple])) |
|
610 |
# now, mapping to k3 should add a k3 leaf
|
|
611 |
prefix, nodes = node.map(None, ('k3',), 'quux') |
|
612 |
self.assertEqual("k", prefix) |
|
613 |
self.assertEqual([("", node)], nodes) |
|
614 |
# check new child details
|
|
615 |
child = node._items['k3'] |
|
616 |
self.assertIsInstance(child, LeafNode) |
|
617 |
self.assertEqual(1, len(child)) |
|
618 |
self.assertEqual({('k3',): 'quux'}, self.to_dict(child, None)) |
|
619 |
self.assertEqual(None, child._key) |
|
620 |
self.assertEqual(10, child.maximum_size) |
|
621 |
self.assertEqual(1, child._key_width) |
|
622 |
# Check overall structure:
|
|
623 |
self.assertEqual(3, len(chkmap)) |
|
624 |
self.assertEqual({('k1',): 'foo', ('k2',): 'bar', ('k3',): 'quux'}, |
|
625 |
self.to_dict(chkmap)) |
|
626 |
# serialising should only serialise the new data - k3 and the internal
|
|
627 |
# node.
|
|
628 |
keys = list(node.serialise(chkmap._store)) |
|
629 |
child_key = child.serialise(chkmap._store)[0] |
|
630 |
self.assertEqual([child_key, keys[1]], keys) |
|
631 |
||
632 |
def test_map_to_child_child_splits_new(self): |
|
633 |
chkmap = self._get_map({('k1',):'foo', ('k22',):'bar'}, maximum_size=10) |
|
634 |
# Check for the canonical root value for this tree:
|
|
635 |
self.assertEqual(('sha1:d3f06fc03d8f50845894d8d04cc5a3f47e62948d',), |
|
636 |
chkmap._root_node) |
|
637 |
chkmap._ensure_root() |
|
638 |
node = chkmap._root_node |
|
639 |
# Ensure test validity: nothing paged in below the root.
|
|
640 |
self.assertEqual(2, |
|
641 |
len([value for value in node._items.values() |
|
642 |
if type(value) == tuple])) |
|
643 |
# now, mapping to k23 causes k22 ('k2' in node) to split into k22 and
|
|
644 |
# k23, which for simplicity in the current implementation generates
|
|
645 |
# a new internal node between node, and k22/k23.
|
|
646 |
prefix, nodes = node.map(chkmap._store, ('k23',), 'quux') |
|
647 |
self.assertEqual("k", prefix) |
|
648 |
self.assertEqual([("", node)], nodes) |
|
649 |
# check new child details
|
|
650 |
child = node._items['k2'] |
|
651 |
self.assertIsInstance(child, InternalNode) |
|
652 |
self.assertEqual(2, len(child)) |
|
653 |
self.assertEqual({('k22',): 'bar', ('k23',): 'quux'}, |
|
654 |
self.to_dict(child, None)) |
|
655 |
self.assertEqual(None, child._key) |
|
656 |
self.assertEqual(10, child.maximum_size) |
|
657 |
self.assertEqual(1, child._key_width) |
|
658 |
self.assertEqual(3, child._node_width) |
|
659 |
# Check overall structure:
|
|
660 |
self.assertEqual(3, len(chkmap)) |
|
661 |
self.assertEqual({('k1',): 'foo', ('k22',): 'bar', ('k23',): 'quux'}, |
|
662 |
self.to_dict(chkmap)) |
|
663 |
# serialising should only serialise the new data - although k22 hasn't
|
|
664 |
# changed because its a special corner case (splitting on with only one
|
|
665 |
# key leaves one node unaltered), in general k22 is serialised, so we
|
|
666 |
# expect k22, k23, the new internal node, and node, to be serialised.
|
|
667 |
keys = list(node.serialise(chkmap._store)) |
|
668 |
child_key = child._key |
|
669 |
k22_key = child._items['k22']._key |
|
670 |
k23_key = child._items['k23']._key |
|
671 |
self.assertEqual([k22_key, k23_key, child_key, keys[-1]], keys) |
|
672 |
self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',), |
|
673 |
keys[-1]) |
|
674 |
||
675 |
def test_unmap_k23_from_k1_k22_k23_gives_k1_k22_tree_new(self): |
|
676 |
chkmap = self._get_map( |
|
677 |
{('k1',):'foo', ('k22',):'bar', ('k23',): 'quux'}, maximum_size=10) |
|
678 |
# Check we have the expected tree.
|
|
679 |
self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',), |
|
680 |
chkmap._root_node) |
|
681 |
chkmap._ensure_root() |
|
682 |
node = chkmap._root_node |
|
683 |
# unmapping k23 should give us a root, with k1 and k22 as direct
|
|
684 |
# children.
|
|
685 |
result = node.unmap(chkmap._store, ('k23',)) |
|
686 |
# check the pointed-at object within node - k2 should now point at the
|
|
687 |
# k22 leaf (which should not even have been paged in).
|
|
688 |
ptr = node._items['k2'] |
|
689 |
self.assertIsInstance(ptr, tuple) |
|
690 |
child = _deserialise(self.read_bytes(chkmap._store, ptr), ptr) |
|
691 |
self.assertIsInstance(child, LeafNode) |
|
692 |
self.assertEqual(1, len(child)) |
|
693 |
self.assertEqual({('k22',): 'bar'}, |
|
694 |
self.to_dict(child, None)) |
|
695 |
# Check overall structure is instact:
|
|
696 |
self.assertEqual(2, len(chkmap)) |
|
697 |
self.assertEqual({('k1',): 'foo', ('k22',): 'bar'}, |
|
698 |
self.to_dict(chkmap)) |
|
699 |
# serialising should only serialise the new data - the root node.
|
|
700 |
keys = list(node.serialise(chkmap._store)) |
|
701 |
self.assertEqual([keys[-1]], keys) |
|
702 |
self.assertEqual(('sha1:d3f06fc03d8f50845894d8d04cc5a3f47e62948d',), keys[-1]) |
|
|
3735.2.18
by Robert Collins
Partial multi-layer chk dictionary trees. |
703 |
|
|
3735.2.23
by Robert Collins
Test unmapping with one child left but multiple keys. |
704 |
def test_unmap_k1_from_k1_k22_k23_gives_k22_k23_tree_new(self): |
705 |
chkmap = self._get_map( |
|
706 |
{('k1',):'foo', ('k22',):'bar', ('k23',): 'quux'}, maximum_size=10) |
|
707 |
# Check we have the expected tree.
|
|
708 |
self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',), |
|
709 |
chkmap._root_node) |
|
710 |
chkmap._ensure_root() |
|
711 |
node = chkmap._root_node |
|
712 |
k2_ptr = node._items['k2'] |
|
713 |
# unmapping k21 should give us a root, with k22 and k23 as direct
|
|
714 |
# children, and should not have needed to page in the subtree.
|
|
715 |
result = node.unmap(chkmap._store, ('k1',)) |
|
716 |
self.assertEqual(k2_ptr, result) |
|
717 |
||
|
3735.2.21
by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux. |
718 |
|
719 |
# leaf:
|
|
720 |
# map -> fits - done
|
|
721 |
# map -> doesn't fit - shrink from left till fits
|
|
722 |
# key data to return: the common prefix, new nodes.
|
|
723 |
||
724 |
# unmap -> how to tell if siblings can be combined.
|
|
725 |
# combing leaf nodes means expanding the prefix to the left; so gather the size of
|
|
726 |
# all the leaf nodes addressed by expanding the prefix by 1; if any adjacent node
|
|
727 |
# is an internal node, we know that that is a dense subtree - can't combine.
|
|
728 |
# otherwise as soon as the sum of serialised values exceeds the split threshold
|
|
729 |
# we know we can't combine - stop.
|
|
730 |
# unmap -> key return data - space in node, common prefix length? and key count
|
|
731 |
# internal:
|
|
732 |
# variable length prefixes? -> later start with fixed width to get something going
|
|
733 |
# map -> fits - update pointer to leaf
|
|
734 |
# return [prefix and node] - seems sound.
|
|
735 |
# map -> doesn't fit - find unique prefix and shift right
|
|
736 |
# create internal nodes for all the partitions, return list of unique
|
|
737 |
# prefixes and nodes.
|
|
738 |
# map -> new prefix - create a leaf
|
|
739 |
# unmap -> if child key count 0, remove
|
|
740 |
# unmap -> return space in node, common prefix length? (why?), key count
|
|
741 |
# map:
|
|
742 |
# map, if 1 node returned, use it, otherwise make an internal and populate.
|
|
743 |
# map - unmap - if empty, use empty leafnode (avoids special cases in driver
|
|
744 |
# code)
|
|
745 |
# map inits as empty leafnode.
|
|
746 |
# tools:
|
|
747 |
# visualiser
|
|
748 |
||
749 |
||
750 |
# how to handle:
|
|
751 |
# AA, AB, AC, AD, BA
|
|
752 |
# packed internal node - ideal:
|
|
753 |
# AA, AB, AC, AD, BA
|
|
754 |
# single byte fanout - A,B, AA,AB,AC,AD, BA
|
|
755 |
# build order's:
|
|
756 |
# BA
|
|
757 |
# AB - split, but we want to end up with AB, BA, in one node, with
|
|
758 |
# 1-4K get0
|
|
|
3735.9.1
by John Arbash Meinel
Start working on an iter_interesting_nodes, which can find nodes to transmit |
759 |
|
760 |
||
761 |
class TestIterInterestingNodes(TestCaseWithStore): |
|
762 |
||
763 |
def get_chk_bytes(self): |
|
764 |
if getattr(self, '_chk_bytes', None) is None: |
|
765 |
self._chk_bytes = super(TestIterInterestingNodes, |
|
766 |
self).get_chk_bytes() |
|
767 |
return self._chk_bytes |
|
768 |
||
769 |
def get_map_key(self, a_dict): |
|
770 |
c_map = self._get_map(a_dict, maximum_size=10, |
|
771 |
chk_bytes=self.get_chk_bytes()) |
|
772 |
return c_map.key() |
|
773 |
||
774 |
def assertIterInteresting(self, expected, interesting_keys, |
|
775 |
uninteresting_keys): |
|
776 |
"""Check the result of iter_interesting_nodes. |
|
777 |
||
778 |
:param expected: A list of (record_keys, interesting_chk_pages,
|
|
779 |
interesting key value pairs)
|
|
780 |
"""
|
|
781 |
store = self.get_chk_bytes() |
|
782 |
iter_nodes = chk_map.iter_interesting_nodes(store, interesting_keys, |
|
783 |
uninteresting_keys) |
|
784 |
for count, (exp, act) in enumerate(izip(expected, iter_nodes)): |
|
785 |
exp_record_keys, exp_chks, exp_items = exp |
|
786 |
records, chks, items = act |
|
787 |
exp_tuple = (sorted(exp_record_keys), sorted(exp_chks), items) |
|
788 |
act_tuple = (sorted(records.keys()), sorted(chks), items) |
|
789 |
self.assertEqual(exp_tuple, act_tuple) |
|
790 |
self.assertEqual(len(expected), count + 1) |
|
791 |
||
792 |
def test_none_to_one_key(self): |
|
793 |
basis = self.get_map_key({}) |
|
794 |
target = self.get_map_key({('a',): 'content'}) |
|
795 |
self.assertIterInteresting( |
|
796 |
[([target], [target], [(('a',), 'content')])], |
|
797 |
[target], [basis]) |
|
798 |
||
799 |
def test_one_to_none_key(self): |
|
800 |
basis = self.get_map_key({('a',): 'content'}) |
|
801 |
target = self.get_map_key({}) |
|
802 |
self.assertIterInteresting( |
|
803 |
[([target], [target], [])], |
|
804 |
[target], [basis]) |
|
805 |
||
806 |
def test_common_pages(self): |
|
807 |
basis = self.get_map_key({('a',): 'content', |
|
808 |
('b',): 'content', |
|
809 |
('c',): 'content', |
|
810 |
})
|
|
811 |
target = self.get_map_key({('a',): 'content', |
|
812 |
('b',): 'other content', |
|
813 |
('c',): 'content', |
|
814 |
})
|
|
815 |
# Is there a way to get this more directly?
|
|
816 |
b_key = ('sha1:1d7a45ded01ab77c069350c0e290ae34db5b549b',) |
|
817 |
# This should return the root node, and the node for the 'b' key
|
|
818 |
self.assertIterInteresting( |
|
819 |
[([target], [target], []), |
|
820 |
([b_key], [b_key], [(('b',), 'other content')])], |
|
821 |
[target], [basis]) |
|
822 |
||
823 |
def test_common_sub_page(self): |
|
824 |
basis = self.get_map_key({('aaa',): 'common', |
|
825 |
('c',): 'common', |
|
826 |
})
|
|
827 |
target = self.get_map_key({('aaa',): 'common', |
|
828 |
('aab',): 'new', |
|
829 |
('c',): 'common', |
|
830 |
})
|
|
831 |
# The key for the internal aa node
|
|
832 |
aa_key = ('sha1:2ce01860338a614b93883a5bbeb89920137ac7ef',) |
|
833 |
# The key for the leaf aab node
|
|
834 |
aab_key = ('sha1:10567a3bfcc764fb8d8d9edaa28c0934ada366c5',) |
|
835 |
self.assertIterInteresting( |
|
836 |
[([target], [target], []), |
|
837 |
([aa_key], [aa_key], []), |
|
838 |
([aab_key], [aab_key], [(('aab',), 'new')])], |
|
839 |
[target], [basis]) |
|
840 |
||
841 |
def test_multiple_maps(self): |
|
842 |
basis1 = self.get_map_key({('aaa',): 'common', |
|
843 |
('aab',): 'basis1', |
|
844 |
})
|
|
845 |
basis2 = self.get_map_key({('bbb',): 'common', |
|
846 |
('bbc',): 'basis2', |
|
847 |
})
|
|
848 |
target1 = self.get_map_key({('aaa',): 'common', |
|
849 |
('aac',): 'target1', |
|
850 |
('bbb',): 'common', |
|
851 |
})
|
|
852 |
target2 = self.get_map_key({('aaa',): 'common', |
|
853 |
('bba',): 'target2', |
|
854 |
('bbb',): 'common', |
|
855 |
})
|
|
856 |
# The key for the target1 internal aa node
|
|
857 |
aa_key = ('sha1:4c6b1e3e6ecb68fe039d2b00c9091bc037ebf203',) |
|
858 |
# The key for the leaf aac node
|
|
859 |
aac_key = ('sha1:8089f6b4f3bd2a058c41be199ef5af0c5b9a0c4f',) |
|
860 |
# The key for the target2 internal bb node
|
|
861 |
bb_key = ('sha1:5ce6a69a21060222bb0a5b48fdbfcca586cc9183',) |
|
862 |
# The key for the leaf bba node
|
|
863 |
bba_key = () |
|
864 |
import pdb; pdb.set_trace() |
|
865 |
self.assertIterInteresting( |
|
866 |
[([target1, target2], [target1, target2], []), |
|
867 |
([aa_key, bb_key], [aa_key, bb_key], []), |
|
868 |
([aac_key, bba_key], [aac_key, bba_key], |
|
869 |
[(('aac',), 'target1'), (('bba',), 'target2')]), |
|
870 |
], [target1, target2], [basis1, basis2]) |