/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
1
# Copyright (C) 2008 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Tests for maps built on a CHK versionedfiles facility."""
18
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
19
from bzrlib.chk_map import (
20
    CHKMap,
21
    InternalNode,
22
    LeafNode,
23
    _deserialise,
24
    )
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
25
from bzrlib.tests import TestCaseWithTransport
26
27
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
28
class TestCaseWithStore(TestCaseWithTransport):
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
29
30
    def get_chk_bytes(self):
31
        # The eassiest way to get a CHK store is a development3 repository and
32
        # then work with the chk_bytes attribute directly.
33
        repo = self.make_repository(".", format="development3")
34
        repo.lock_write()
35
        self.addCleanup(repo.unlock)
36
        repo.start_write_group()
37
        self.addCleanup(repo.abort_write_group)
38
        return repo.chk_bytes
39
3735.2.30 by Robert Collins
Start iter_changes between CHKMap instances.
40
    def _get_map(self, a_dict, maximum_size=0, chk_bytes=None):
41
        if chk_bytes is None:
42
            chk_bytes = self.get_chk_bytes()
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
43
        root_key = CHKMap.from_dict(chk_bytes, a_dict, maximum_size=maximum_size)
44
        chkmap = CHKMap(chk_bytes, root_key)
45
        return chkmap
46
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
47
    def read_bytes(self, chk_bytes, key):
48
        stream = chk_bytes.get_record_stream([key], 'unordered', True)
49
        return stream.next().get_bytes_as("fulltext")
50
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
51
    def to_dict(self, node, *args):
52
        return dict(node.iteritems(*args))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
53
54
55
class TestMap(TestCaseWithStore):
56
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
57
    def assertHasABMap(self, chk_bytes):
3735.2.24 by Robert Collins
test_chk_map tests all passing.
58
        root_key = ('sha1:f14dd34def95036bc06bb5c0ed95437d7383a04a',)
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
59
        self.assertEqual(
3735.2.24 by Robert Collins
test_chk_map tests all passing.
60
            'chkleaf:\n0\n1\n1\na\x00b\n',
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
61
            self.read_bytes(chk_bytes, root_key))
3735.2.24 by Robert Collins
test_chk_map tests all passing.
62
        return root_key
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
63
64
    def assertHasEmptyMap(self, chk_bytes):
3735.2.24 by Robert Collins
test_chk_map tests all passing.
65
        root_key = ('sha1:4e6482a3a5cb2d61699971ac77befe11a0ec5779',)
66
        self.assertEqual("chkleaf:\n0\n1\n0\n", self.read_bytes(chk_bytes, root_key))
67
        return root_key
3735.2.17 by Robert Collins
Cache node length to avoid full iteration on __len__ calls.
68
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
69
    def test_from_dict_empty(self):
70
        chk_bytes = self.get_chk_bytes()
71
        root_key = CHKMap.from_dict(chk_bytes, {})
3735.2.24 by Robert Collins
test_chk_map tests all passing.
72
        # Check the data was saved and inserted correctly.
73
        expected_root_key = self.assertHasEmptyMap(chk_bytes)
74
        self.assertEqual(expected_root_key, root_key)
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
75
76
    def test_from_dict_ab(self):
77
        chk_bytes = self.get_chk_bytes()
78
        root_key = CHKMap.from_dict(chk_bytes, {"a":"b"})
3735.2.24 by Robert Collins
test_chk_map tests all passing.
79
        # Check the data was saved and inserted correctly.
80
        expected_root_key = self.assertHasABMap(chk_bytes)
81
        self.assertEqual(expected_root_key, root_key)
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
82
83
    def test_apply_empty_ab(self):
84
        # applying a delta (None, "a", "b") to an empty chkmap generates the
85
        # same map as from_dict_ab.
86
        chk_bytes = self.get_chk_bytes()
87
        root_key = CHKMap.from_dict(chk_bytes, {})
88
        chkmap = CHKMap(chk_bytes, root_key)
89
        new_root = chkmap.apply_delta([(None, "a", "b")])
3735.2.24 by Robert Collins
test_chk_map tests all passing.
90
        # Check the data was saved and inserted correctly.
91
        expected_root_key = self.assertHasABMap(chk_bytes)
92
        self.assertEqual(expected_root_key, new_root)
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
93
        # The update should have left us with an in memory root node, with an
94
        # updated key.
95
        self.assertEqual(new_root, chkmap._root_node._key)
96
97
    def test_apply_ab_empty(self):
3735.2.30 by Robert Collins
Start iter_changes between CHKMap instances.
98
        # applying a delta ("a", None, None) to a map with 'a' in it generates
99
        # an empty map.
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
100
        chk_bytes = self.get_chk_bytes()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
101
        root_key = CHKMap.from_dict(chk_bytes, {("a",):"b"})
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
102
        chkmap = CHKMap(chk_bytes, root_key)
3735.2.24 by Robert Collins
test_chk_map tests all passing.
103
        new_root = chkmap.apply_delta([(("a",), None, None)])
104
        # Check the data was saved and inserted correctly.
105
        expected_root_key = self.assertHasEmptyMap(chk_bytes)
106
        self.assertEqual(expected_root_key, new_root)
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
107
        # The update should have left us with an in memory root node, with an
108
        # updated key.
109
        self.assertEqual(new_root, chkmap._root_node._key)
110
3735.2.30 by Robert Collins
Start iter_changes between CHKMap instances.
111
    def test_iter_changes_empty_ab(self):
112
        # Asking for changes between an empty dict to a dict with keys returns
113
        # all the keys.
114
        basis = self._get_map({})
115
        target = self._get_map(
116
            {('a',): 'content here', ('b',): 'more content'},
117
            chk_bytes=basis._store)
118
        self.assertEqual([(('a',), None, 'content here'),
119
            (('b',), None, 'more content')], list(target.iter_changes(basis)))
120
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
121
    def test_iteritems_empty(self):
122
        chk_bytes = self.get_chk_bytes()
123
        root_key = CHKMap.from_dict(chk_bytes, {})
124
        chkmap = CHKMap(chk_bytes, root_key)
125
        self.assertEqual([], list(chkmap.iteritems()))
126
127
    def test_iteritems_two_items(self):
128
        chk_bytes = self.get_chk_bytes()
129
        root_key = CHKMap.from_dict(chk_bytes,
130
            {"a":"content here", "b":"more content"})
131
        chkmap = CHKMap(chk_bytes, root_key)
3735.2.24 by Robert Collins
test_chk_map tests all passing.
132
        self.assertEqual([(("a",), "content here"), (("b",), "more content")],
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
133
            sorted(list(chkmap.iteritems())))
134
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
135
    def test_iteritems_selected_one_of_two_items(self):
3735.2.24 by Robert Collins
test_chk_map tests all passing.
136
        chkmap = self._get_map( {("a",):"content here", ("b",):"more content"})
137
        self.assertEqual({("a",): "content here"},
138
            self.to_dict(chkmap, [("a",)]))
3735.2.9 by Robert Collins
Get a working chk_map using inventory implementation bootstrapped.
139
3735.2.17 by Robert Collins
Cache node length to avoid full iteration on __len__ calls.
140
    def test___len__empty(self):
141
        chkmap = self._get_map({})
142
        self.assertEqual(0, len(chkmap))
143
144
    def test___len__2(self):
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
145
        chkmap = self._get_map({"foo":"bar", "gam":"quux"})
146
        self.assertEqual(2, len(chkmap))
147
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
148
    def test_max_size_100_bytes_new(self):
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
149
        # When there is a 100 byte upper node limit, a tree is formed.
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
150
        chkmap = self._get_map({("k1"*50,):"v1", ("k2"*50,):"v2"}, maximum_size=100)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
151
        # We expect three nodes:
152
        # A root, with two children, and with two key prefixes - k1 to one, and
153
        # k2 to the other as our node splitting is only just being developed.
154
        # The maximum size should be embedded
155
        chkmap._ensure_root()
156
        self.assertEqual(100, chkmap._root_node.maximum_size)
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
157
        self.assertEqual(1, chkmap._root_node._key_width)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
158
        # There should be two child nodes, and prefix of 2(bytes):
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
159
        self.assertEqual(2, len(chkmap._root_node._items))
160
        self.assertEqual("k", chkmap._root_node.unique_serialised_prefix())
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
161
        # The actual nodes pointed at will change as serialisers change; so
162
        # here we test that the key prefix is correct; then load the nodes and
163
        # check they have the right pointed at key; whether they have the
164
        # pointed at value inline or not is also unrelated to this test so we
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
165
        # don't check that in detail - rather we just check the aggregate
166
        # value.
167
        nodes = sorted(chkmap._root_node._items.items())
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
168
        ptr1 = nodes[0]
169
        ptr2 = nodes[1]
170
        self.assertEqual('k1', ptr1[0])
171
        self.assertEqual('k2', ptr2[0])
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
172
        node1 = _deserialise(chkmap._read_bytes(ptr1[1]), ptr1[1])
173
        self.assertIsInstance(node1, LeafNode)
174
        self.assertEqual(1, len(node1))
175
        self.assertEqual({('k1'*50,): 'v1'}, self.to_dict(node1, chkmap._store))
176
        node2 = _deserialise(chkmap._read_bytes(ptr2[1]), ptr2[1])
177
        self.assertIsInstance(node2, LeafNode)
178
        self.assertEqual(1, len(node2))
179
        self.assertEqual({('k2'*50,): 'v2'}, self.to_dict(node2, chkmap._store))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
180
        # Having checked we have a good structure, check that the content is
181
        # still accessible.
182
        self.assertEqual(2, len(chkmap))
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
183
        self.assertEqual({("k1"*50,): "v1", ("k2"*50,): "v2"},
184
            self.to_dict(chkmap))
185
186
    def test_init_root_is_LeafNode_new(self):
187
        chk_bytes = self.get_chk_bytes()
188
        chkmap = CHKMap(chk_bytes, None)
189
        self.assertIsInstance(chkmap._root_node, LeafNode)
190
        self.assertEqual({}, self.to_dict(chkmap))
191
        self.assertEqual(0, len(chkmap))
192
193
    def test_init_and_save_new(self):
194
        chk_bytes = self.get_chk_bytes()
195
        chkmap = CHKMap(chk_bytes, None)
196
        key = chkmap._save()
197
        leaf_node = LeafNode()
198
        self.assertEqual([key], leaf_node.serialise(chk_bytes))
199
200
    def test_map_first_item_new(self):
201
        chk_bytes = self.get_chk_bytes()
202
        chkmap = CHKMap(chk_bytes, None)
203
        chkmap.map(("foo,",), "bar")
204
        self.assertEqual({('foo,',): 'bar'}, self.to_dict(chkmap))
205
        self.assertEqual(1, len(chkmap))
206
        key = chkmap._save()
207
        leaf_node = LeafNode()
208
        leaf_node.map(chk_bytes, ("foo,",), "bar")
209
        self.assertEqual([key], leaf_node.serialise(chk_bytes))
210
211
    def test_unmap_last_item_root_is_leaf_new(self):
212
        chkmap = self._get_map({("k1"*50,): "v1", ("k2"*50,): "v2"})
213
        chkmap.unmap(("k1"*50,))
214
        chkmap.unmap(("k2"*50,))
215
        self.assertEqual(0, len(chkmap))
216
        self.assertEqual({}, self.to_dict(chkmap))
217
        key = chkmap._save()
218
        leaf_node = LeafNode()
219
        self.assertEqual([key], leaf_node.serialise(chkmap._store))
3735.2.17 by Robert Collins
Cache node length to avoid full iteration on __len__ calls.
220
3735.2.8 by Robert Collins
New chk_map module for use in tree based inventory storage.
221
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
222
class TestLeafNode(TestCaseWithStore):
223
224
    def test_current_size_empty(self):
225
        node = LeafNode()
226
        self.assertEqual(15, node._current_size())
227
228
    def test_current_size_size_changed(self):
229
        node = LeafNode()
230
        node.set_maximum_size(10)
231
        self.assertEqual(16, node._current_size())
232
233
    def test_current_size_width_changed(self):
234
        node = LeafNode()
235
        node._key_width = 10
236
        self.assertEqual(16, node._current_size())
237
238
    def test_current_size_items(self):
239
        node = LeafNode()
240
        base_size = node._current_size()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
241
        node.map(None, ("foo bar",), "baz")
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
242
        self.assertEqual(base_size + 12, node._current_size())
243
244
    def test_deserialise_empty(self):
245
        node = LeafNode.deserialise("chkleaf:\n10\n1\n0\n", ("sha1:1234",))
246
        self.assertEqual(0, len(node))
247
        self.assertEqual(10, node.maximum_size)
248
        self.assertEqual(("sha1:1234",), node.key())
249
250
    def test_deserialise_items(self):
251
        node = LeafNode.deserialise(
252
            "chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",))
253
        self.assertEqual(2, len(node))
254
        self.assertEqual([(("foo bar",), "baz"), (("quux",), "blarh")],
3735.2.24 by Robert Collins
test_chk_map tests all passing.
255
            sorted(node.iteritems(None)))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
256
3735.2.25 by Robert Collins
CHKInventory core tests passing.
257
    def test_deserialise_item_with_null_width_1(self):
258
        node = LeafNode.deserialise(
259
            "chkleaf:\n0\n1\n2\nfoo\x00bar\x00baz\nquux\x00blarh\n",
260
            ("sha1:1234",))
261
        self.assertEqual(2, len(node))
262
        self.assertEqual([(("foo",), "bar\x00baz"), (("quux",), "blarh")],
263
            sorted(node.iteritems(None)))
264
265
    def test_deserialise_item_with_null_width_2(self):
266
        node = LeafNode.deserialise(
267
            "chkleaf:\n0\n2\n2\nfoo\x001\x00bar\x00baz\nquux\x00\x00blarh\n",
268
            ("sha1:1234",))
269
        self.assertEqual(2, len(node))
270
        self.assertEqual([(("foo", "1"), "bar\x00baz"), (("quux", ""), "blarh")],
271
            sorted(node.iteritems(None)))
272
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
273
    def test_iteritems_selected_one_of_two_items(self):
274
        node = LeafNode.deserialise(
275
            "chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",))
276
        self.assertEqual(2, len(node))
277
        self.assertEqual([(("quux",), "blarh")],
3735.2.24 by Robert Collins
test_chk_map tests all passing.
278
            sorted(node.iteritems(None, [("quux",), ("qaz",)])))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
279
280
    def test_key_new(self):
281
        node = LeafNode()
282
        self.assertEqual(None, node.key())
283
284
    def test_key_after_map(self):
285
        node = LeafNode.deserialise("chkleaf:\n10\n1\n0\n", ("sha1:1234",))
3735.2.24 by Robert Collins
test_chk_map tests all passing.
286
        node.map(None, ("foo bar",), "baz quux")
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
287
        self.assertEqual(None, node.key())
288
289
    def test_key_after_unmap(self):
290
        node = LeafNode.deserialise(
291
            "chkleaf:\n0\n1\n2\nfoo bar\x00baz\nquux\x00blarh\n", ("sha1:1234",))
3735.2.24 by Robert Collins
test_chk_map tests all passing.
292
        node.unmap(None, ("foo bar",))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
293
        self.assertEqual(None, node.key())
294
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
295
    def test_map_exceeding_max_size_only_entry_new(self):
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
296
        node = LeafNode()
297
        node.set_maximum_size(10)
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
298
        result = node.map(None, ("foo bar",), "baz quux")
299
        self.assertEqual(("foo bar", [("", node)]), result)
300
        self.assertTrue(10 < node._current_size())
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
301
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
302
    def test_map_exceeding_max_size_second_entry_early_difference_new(self):
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
303
        node = LeafNode()
304
        node.set_maximum_size(10)
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
305
        node.map(None, ("foo bar",), "baz quux")
306
        prefix, result = list(node.map(None, ("blue",), "red"))
307
        self.assertEqual("", prefix)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
308
        self.assertEqual(2, len(result))
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
309
        split_chars = set([result[0][0], result[1][0]])
310
        self.assertEqual(set(["f", "b"]), split_chars)
311
        nodes = dict(result)
312
        node = nodes["f"]
313
        self.assertEqual({("foo bar",): "baz quux"}, self.to_dict(node, None))
314
        self.assertEqual(10, node.maximum_size)
315
        self.assertEqual(1, node._key_width)
316
        node = nodes["b"]
317
        self.assertEqual({("blue",): "red"}, self.to_dict(node, None))
318
        self.assertEqual(10, node.maximum_size)
319
        self.assertEqual(1, node._key_width)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
320
321
    def test_map_first(self):
322
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
323
        result = node.map(None, ("foo bar",), "baz quux")
324
        self.assertEqual(("foo bar", [("", node)]), result)
325
        self.assertEqual({("foo bar",):"baz quux"}, self.to_dict(node, None))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
326
        self.assertEqual(1, len(node))
327
328
    def test_map_second(self):
329
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
330
        node.map(None, ("foo bar",), "baz quux")
331
        result = node.map(None, ("bingo",), "bango")
332
        self.assertEqual(("", [("", node)]), result)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
333
        self.assertEqual({("foo bar",):"baz quux", ("bingo",):"bango"},
3735.2.24 by Robert Collins
test_chk_map tests all passing.
334
            self.to_dict(node, None))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
335
        self.assertEqual(2, len(node))
336
337
    def test_map_replacement(self):
338
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
339
        node.map(None, ("foo bar",), "baz quux")
340
        result = node.map(None, ("foo bar",), "bango")
341
        self.assertEqual(("foo bar", [("", node)]), result)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
342
        self.assertEqual({("foo bar",): "bango"},
3735.2.24 by Robert Collins
test_chk_map tests all passing.
343
            self.to_dict(node, None))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
344
        self.assertEqual(1, len(node))
345
346
    def test_serialise_empty(self):
347
        store = self.get_chk_bytes()
348
        node = LeafNode()
349
        node.set_maximum_size(10)
350
        expected_key = ("sha1:62cc3565b48b0e830216e652cf99c6bd6b05b4b9",)
351
        self.assertEqual([expected_key],
352
            list(node.serialise(store)))
353
        self.assertEqual("chkleaf:\n10\n1\n0\n", self.read_bytes(store, expected_key))
354
        self.assertEqual(expected_key, node.key())
355
356
    def test_serialise_items(self):
357
        store = self.get_chk_bytes()
358
        node = LeafNode()
359
        node.set_maximum_size(10)
3735.2.24 by Robert Collins
test_chk_map tests all passing.
360
        node.map(None, ("foo bar",), "baz quux")
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
361
        expected_key = ("sha1:d44cb6f0299b7e047da7f9e98f810e98f1dce1a7",)
362
        self.assertEqual([expected_key],
363
            list(node.serialise(store)))
364
        self.assertEqual("chkleaf:\n10\n1\n1\nfoo bar\x00baz quux\n",
365
            self.read_bytes(store, expected_key))
366
        self.assertEqual(expected_key, node.key())
367
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
368
    def test_unique_serialised_prefix_empty_new(self):
369
        node = LeafNode()
370
        self.assertEqual("", node.unique_serialised_prefix())
371
372
    def test_unique_serialised_prefix_one_item_new(self):
373
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
374
        node.map(None, ("foo bar", "baz"), "baz quux")
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
375
        self.assertEqual("foo bar\x00baz", node.unique_serialised_prefix())
376
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
377
    def test_unmap_missing(self):
378
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
379
        self.assertRaises(KeyError, node.unmap, None, ("foo bar",))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
380
381
    def test_unmap_present(self):
382
        node = LeafNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
383
        node.map(None, ("foo bar",), "baz quux")
384
        result = node.unmap(None, ("foo bar",))
385
        self.assertEqual(node, result)
386
        self.assertEqual({}, self.to_dict(node, None))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
387
        self.assertEqual(0, len(node))
388
389
390
class TestInternalNode(TestCaseWithStore):
391
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
392
    def test_add_node_empty_new(self):
393
        node = InternalNode()
394
        child = LeafNode()
395
        child.set_maximum_size(100)
396
        child.map(None, ("foo",), "bar")
397
        node.add_node("foo", child)
398
        # Note that node isn't strictly valid now as a tree (only one child),
399
        # but thats ok for this test.
400
        # The first child defines the node's width:
401
        self.assertEqual(3, node._node_width)
402
        # We should be able to iterate over the contents without doing IO.
403
        self.assertEqual({('foo',): 'bar'}, self.to_dict(node, None))
404
        # The length should be known:
405
        self.assertEqual(1, len(node))
406
        # serialising the node should serialise the child and the node.
407
        chk_bytes = self.get_chk_bytes()
408
        keys = list(node.serialise(chk_bytes))
409
        child_key = child.serialise(chk_bytes)[0]
410
        self.assertEqual(
411
            [child_key, ('sha1:db23b260c2bf46bf7446c39f91668900a2491610',)],
412
            keys)
413
        # We should be able to access deserialised content.
414
        bytes = self.read_bytes(chk_bytes, keys[1])
415
        node = _deserialise(bytes, keys[1])
416
        self.assertEqual(1, len(node))
417
        self.assertEqual({('foo',): 'bar'}, self.to_dict(node, chk_bytes))
418
        self.assertEqual(3, node._node_width)
419
420
    def test_add_node_resets_key_new(self):
421
        node = InternalNode()
422
        child = LeafNode()
423
        child.set_maximum_size(100)
424
        child.map(None, ("foo",), "bar")
425
        node.add_node("foo", child)
426
        chk_bytes = self.get_chk_bytes()
427
        keys = list(node.serialise(chk_bytes))
428
        self.assertEqual(keys[1], node._key)
429
        node.add_node("fos", child)
430
        self.assertEqual(None, node._key)
431
432
#    def test_add_node_empty_oversized_one_ok_new(self):
433
#    def test_add_node_one_oversized_second_kept_minimum_fan(self):
434
#    def test_add_node_two_oversized_third_kept_minimum_fan(self):
435
#    def test_add_node_one_oversized_second_splits_errors(self):
436
437
    def test_iteritems_empty_new(self):
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
438
        node = InternalNode()
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
439
        self.assertEqual([], sorted(node.iteritems(None)))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
440
441
    def test_iteritems_two_children(self):
442
        node = InternalNode()
443
        leaf1 = LeafNode()
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
444
        leaf1.map(None, ('foo bar',), 'quux')
445
        leaf2 = LeafNode()
446
        leaf2.map(None, ('strange',), 'beast')
3735.2.24 by Robert Collins
test_chk_map tests all passing.
447
        node.add_node("f", leaf1)
448
        node.add_node("s", leaf2)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
449
        self.assertEqual([(('foo bar',), 'quux'), (('strange',), 'beast')],
3735.2.24 by Robert Collins
test_chk_map tests all passing.
450
            sorted(node.iteritems(None)))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
451
452
    def test_iteritems_two_children_partial(self):
453
        node = InternalNode()
3735.2.24 by Robert Collins
test_chk_map tests all passing.
454
        leaf1 = LeafNode()
455
        leaf1.map(None, ('foo bar',), 'quux')
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
456
        leaf2 = LeafNode()
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
457
        leaf2.map(None, ('strange',), 'beast')
3735.2.24 by Robert Collins
test_chk_map tests all passing.
458
        node.add_node("f", leaf1)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
459
        # This sets up a path that should not be followed - it will error if
460
        # the code tries to.
3735.2.24 by Robert Collins
test_chk_map tests all passing.
461
        node._items['f'] = None
462
        node.add_node("s", leaf2)
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
463
        self.assertEqual([(('strange',), 'beast')],
3735.2.24 by Robert Collins
test_chk_map tests all passing.
464
            sorted(node.iteritems(None, [('strange',), ('weird',)])))
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
465
466
    def test_iteritems_partial_empty(self):
467
        node = InternalNode()
468
        self.assertEqual([], sorted(node.iteritems([('missing',)])))
469
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
470
    def test_map_to_new_child_new(self):
471
        chkmap = self._get_map({('k1',):'foo', ('k2',):'bar'}, maximum_size=10)
472
        chkmap._ensure_root()
473
        node = chkmap._root_node
474
        # Ensure test validity: nothing paged in below the root.
475
        self.assertEqual(2,
476
            len([value for value in node._items.values()
477
                if type(value) == tuple]))
478
        # now, mapping to k3 should add a k3 leaf
479
        prefix, nodes = node.map(None, ('k3',), 'quux')
480
        self.assertEqual("k", prefix)
481
        self.assertEqual([("", node)], nodes)
482
        # check new child details
483
        child = node._items['k3']
484
        self.assertIsInstance(child, LeafNode)
485
        self.assertEqual(1, len(child))
486
        self.assertEqual({('k3',): 'quux'}, self.to_dict(child, None))
487
        self.assertEqual(None, child._key)
488
        self.assertEqual(10, child.maximum_size)
489
        self.assertEqual(1, child._key_width)
490
        # Check overall structure:
491
        self.assertEqual(3, len(chkmap))
492
        self.assertEqual({('k1',): 'foo', ('k2',): 'bar', ('k3',): 'quux'},
493
            self.to_dict(chkmap))
494
        # serialising should only serialise the new data - k3 and the internal
495
        # node.
496
        keys = list(node.serialise(chkmap._store))
497
        child_key = child.serialise(chkmap._store)[0]
498
        self.assertEqual([child_key, keys[1]], keys)
499
500
    def test_map_to_child_child_splits_new(self):
501
        chkmap = self._get_map({('k1',):'foo', ('k22',):'bar'}, maximum_size=10)
502
        # Check for the canonical root value for this tree:
503
        self.assertEqual(('sha1:d3f06fc03d8f50845894d8d04cc5a3f47e62948d',),
504
            chkmap._root_node)
505
        chkmap._ensure_root()
506
        node = chkmap._root_node
507
        # Ensure test validity: nothing paged in below the root.
508
        self.assertEqual(2,
509
            len([value for value in node._items.values()
510
                if type(value) == tuple]))
511
        # now, mapping to k23 causes k22 ('k2' in node) to split into k22 and
512
        # k23, which for simplicity in the current implementation generates
513
        # a new internal node between node, and k22/k23.
514
        prefix, nodes = node.map(chkmap._store, ('k23',), 'quux')
515
        self.assertEqual("k", prefix)
516
        self.assertEqual([("", node)], nodes)
517
        # check new child details
518
        child = node._items['k2']
519
        self.assertIsInstance(child, InternalNode)
520
        self.assertEqual(2, len(child))
521
        self.assertEqual({('k22',): 'bar', ('k23',): 'quux'},
522
            self.to_dict(child, None))
523
        self.assertEqual(None, child._key)
524
        self.assertEqual(10, child.maximum_size)
525
        self.assertEqual(1, child._key_width)
526
        self.assertEqual(3, child._node_width)
527
        # Check overall structure:
528
        self.assertEqual(3, len(chkmap))
529
        self.assertEqual({('k1',): 'foo', ('k22',): 'bar', ('k23',): 'quux'},
530
            self.to_dict(chkmap))
531
        # serialising should only serialise the new data - although k22 hasn't
532
        # changed because its a special corner case (splitting on with only one
533
        # key leaves one node unaltered), in general k22 is serialised, so we
534
        # expect k22, k23, the new internal node, and node, to be serialised.
535
        keys = list(node.serialise(chkmap._store))
536
        child_key = child._key
537
        k22_key = child._items['k22']._key
538
        k23_key = child._items['k23']._key
539
        self.assertEqual([k22_key, k23_key, child_key, keys[-1]], keys)
540
        self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',),
541
            keys[-1])
542
543
    def test_unmap_k23_from_k1_k22_k23_gives_k1_k22_tree_new(self):
544
        chkmap = self._get_map(
545
            {('k1',):'foo', ('k22',):'bar', ('k23',): 'quux'}, maximum_size=10)
546
        # Check we have the expected tree.
547
        self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',),
548
            chkmap._root_node)
549
        chkmap._ensure_root()
550
        node = chkmap._root_node
551
        # unmapping k23 should give us a root, with k1 and k22 as direct
552
        # children.
553
        result = node.unmap(chkmap._store, ('k23',))
554
        # check the pointed-at object within node - k2 should now point at the
555
        # k22 leaf (which should not even have been paged in).
556
        ptr = node._items['k2']
557
        self.assertIsInstance(ptr, tuple)
558
        child = _deserialise(self.read_bytes(chkmap._store, ptr), ptr)
559
        self.assertIsInstance(child, LeafNode)
560
        self.assertEqual(1, len(child))
561
        self.assertEqual({('k22',): 'bar'},
562
            self.to_dict(child, None))
563
        # Check overall structure is instact:
564
        self.assertEqual(2, len(chkmap))
565
        self.assertEqual({('k1',): 'foo', ('k22',): 'bar'},
566
            self.to_dict(chkmap))
567
        # serialising should only serialise the new data - the root node.
568
        keys = list(node.serialise(chkmap._store))
569
        self.assertEqual([keys[-1]], keys)
570
        self.assertEqual(('sha1:d3f06fc03d8f50845894d8d04cc5a3f47e62948d',), keys[-1])
3735.2.18 by Robert Collins
Partial multi-layer chk dictionary trees.
571
3735.2.23 by Robert Collins
Test unmapping with one child left but multiple keys.
572
    def test_unmap_k1_from_k1_k22_k23_gives_k22_k23_tree_new(self):
573
        chkmap = self._get_map(
574
            {('k1',):'foo', ('k22',):'bar', ('k23',): 'quux'}, maximum_size=10)
575
        # Check we have the expected tree.
576
        self.assertEqual(('sha1:d68cd97c95e847d3dc58c05537aa5fdcdf2cf5da',),
577
            chkmap._root_node)
578
        chkmap._ensure_root()
579
        node = chkmap._root_node
580
        k2_ptr = node._items['k2']
581
        # unmapping k21 should give us a root, with k22 and k23 as direct
582
        # children, and should not have needed to page in the subtree.
583
        result = node.unmap(chkmap._store, ('k1',))
584
        self.assertEqual(k2_ptr, result)
585
3735.2.21 by Robert Collins
BROKEN: multi level CHKMap tries, unfinished, subsystem in flux.
586
587
# leaf:
588
# map -> fits - done
589
# map -> doesn't fit - shrink from left till fits
590
#        key data to return: the common prefix, new nodes.
591
592
# unmap -> how to tell if siblings can be combined.
593
#          combing leaf nodes means expanding the prefix to the left; so gather the size of
594
#          all the leaf nodes addressed by expanding the prefix by 1; if any adjacent node
595
#          is an internal node, we know that that is a dense subtree - can't combine.
596
#          otherwise as soon as the sum of serialised values exceeds the split threshold
597
#          we know we can't combine - stop.
598
# unmap -> key return data - space in node, common prefix length? and key count
599
# internal: 
600
# variable length prefixes? -> later start with fixed width to get something going
601
# map -> fits - update pointer to leaf
602
#        return [prefix and node] - seems sound.
603
# map -> doesn't fit - find unique prefix and shift right
604
#        create internal nodes for all the partitions, return list of unique
605
#        prefixes and nodes.
606
# map -> new prefix - create a leaf
607
# unmap -> if child key count 0, remove
608
# unmap -> return space in node, common prefix length? (why?), key count
609
# map:
610
# map, if 1 node returned, use it, otherwise make an internal and populate.
611
# map - unmap - if empty, use empty leafnode (avoids special cases in driver
612
# code)
613
# map inits as empty leafnode.
614
# tools: 
615
# visualiser
616
617
618
# how to handle:
619
# AA, AB, AC, AD, BA
620
# packed internal node - ideal:
621
# AA, AB, AC, AD, BA
622
# single byte fanout - A,B,   AA,AB,AC,AD,     BA
623
# build order's:
624
# BA
625
# AB - split, but we want to end up with AB, BA, in one node, with 
626
# 1-4K get0