/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/chk_map.py

• browse files at revision 6653.6.4
• compare with another revision
• compare with revision 5200.3.2
• download diff
• download tarball
• view history from revision 6653.6.4

Show diffs side-by-side

added

removed

breezy/chk_map.py

56

56

    static_tuple,

57

57

    trace,

58

58

    )

 

59

from .sixish import (

 

60

    viewitems,

 

61

    viewvalues,

 

62

    )

59

63

from .static_tuple import StaticTuple

60

64

 

61

65

# approx 4MB

212

216

        if isinstance(node, InternalNode):

213

217

            # Trigger all child nodes to get loaded

214

218

            list(node._iter_nodes(self._store))

215

 

            for prefix, sub in sorted(node._items.iteritems()):

 

219

            for prefix, sub in sorted(viewitems(node._items)):

216

220

                result.extend(self._dump_tree_node(sub, prefix, indent + '  ',

217

221

                                                   include_keys=include_keys))

218

222

        else:

219

 

            for key, value in sorted(node._items.iteritems()):

 

223

            for key, value in sorted(viewitems(node._items)):

220

224

                # Don't use prefix nor indent here to line up when used in

221

225

                # tests in conjunction with assertEqualDiff

222

226

                result.append('      %r %r' % (tuple(key), value))

255

259

        result._root_node.set_maximum_size(maximum_size)

256

260

        result._root_node._key_width = key_width

257

261

        delta = []

258

 

        for key, value in initial_value.items():

 

262

        for key, value in viewitems(initial_value):

259

263

            delta.append((None, key, value))

260

264

        root_key = result.apply_delta(delta)

261

265

        return root_key

267

271

        node.set_maximum_size(maximum_size)

268

272

        node._key_width = key_width

269

273

        as_st = StaticTuple.from_sequence

270

 

        node._items = dict([(as_st(key), val) for key, val

271

 

                                               in initial_value.iteritems()])

272

 

        node._raw_size = sum([node._key_value_len(key, value)

273

 

                              for key,value in node._items.iteritems()])

 

274

        node._items = dict((as_st(key), val)

 

275

            for key, val in viewitems(initial_value))

 

276

        node._raw_size = sum(node._key_value_len(key, value)

 

277

            for key, value in viewitems(node._items))

274

278

        node._len = len(node._items)

275

279

        node._compute_search_prefix()

276

280

        node._compute_serialised_prefix()

333

337

            node = a_map._get_node(node)

334

338

            if isinstance(node, LeafNode):

335

339

                path = (node._key, path)

336

 

                for key, value in node._items.items():

 

340

                for key, value in viewitems(node._items):

337

341

                    # For a LeafNode, the key is a serialized_key, rather than

338

342

                    # a search_key, but the heap is using search_keys

339

343

                    search_key = node._search_key_func(key)

341

345

            else:

342

346

                # type(node) == InternalNode

343

347

                path = (node._key, path)

344

 

                for prefix, child in node._items.items():

 

348

                for prefix, child in viewitems(node._items):

345

349

                    heapq.heappush(pending, (prefix, None, child, path))

346

350

        def process_common_internal_nodes(self_node, basis_node):

347

 

            self_items = set(self_node._items.items())

348

 

            basis_items = set(basis_node._items.items())

 

351

            self_items = set(viewitems(self_node._items))

 

352

            basis_items = set(viewitems(basis_node._items))

349

353

            path = (self_node._key, None)

350

354

            for prefix, child in self_items - basis_items:

351

355

                heapq.heappush(self_pending, (prefix, None, child, path))

353

357

            for prefix, child in basis_items - self_items:

354

358

                heapq.heappush(basis_pending, (prefix, None, child, path))

355

359

        def process_common_leaf_nodes(self_node, basis_node):

356

 

            self_items = set(self_node._items.items())

357

 

            basis_items = set(basis_node._items.items())

 

360

            self_items = set(viewitems(self_node._items))

 

361

            basis_items = set(viewitems(basis_node._items))

358

362

            path = (self_node._key, None)

359

363

            for key, value in self_items - basis_items:

360

364

                prefix = self._search_key_func(key)

766

770

                        pass

767

771

                else:

768

772

                    # Short items, we need to match based on a prefix

769

 

                    length_filter = filters.setdefault(len(key), set())

770

 

                    length_filter.add(key)

 

773

                    filters.setdefault(len(key), set()).add(key)

771

774

            if filters:

772

 

                filters = filters.items()

773

 

                for item in self._items.iteritems():

774

 

                    for length, length_filter in filters:

 

775

                filters_itemview = viewitems(filters)

 

776

                for item in viewitems(self._items):

 

777

                    for length, length_filter in filters_itemview:

775

778

                        if item[0][:length] in length_filter:

776

779

                            yield item

777

780

                            break

778

781

        else:

779

 

            for item in self._items.iteritems():

 

782

            for item in viewitems(self._items):

780

783

                yield item

781

784

 

782

785

    def _key_value_len(self, key, value):

838

841

        common_prefix = self._search_prefix

839

842

        split_at = len(common_prefix) + 1

840

843

        result = {}

841

 

        for key, value in self._items.iteritems():

 

844

        for key, value in viewitems(self._items):

842

845

            search_key = self._search_key(key)

843

846

            prefix = search_key[:split_at]

844

847

            # TODO: Generally only 1 key can be exactly the right length,

871

874

                for split, node in node_details:

872

875

                    new_node.add_node(split, node)

873

876

                result[prefix] = new_node

874

 

        return common_prefix, result.items()

 

877

        return common_prefix, list(viewitems(result))

875

878

 

876

879

    def map(self, store, key, value):

877

880

        """Map key to value."""

906

909

        else:

907

910

            lines.append('%s\n' % (self._common_serialised_prefix,))

908

911

            prefix_len = len(self._common_serialised_prefix)

909

 

        for key, value in sorted(self._items.items()):

 

912

        for key, value in sorted(viewitems(self._items)):

910

913

            # Always add a final newline

911

914

            value_lines = osutils.chunks_to_lines([value + '\n'])

912

915

            serialized = "%s\x00%s\n" % (self._serialise_key(key),

1071

1074

            # yielding all nodes, yield whatever we have, and queue up a read

1072

1075

            # for whatever we are missing

1073

1076

            shortcut = True

1074

 

            for prefix, node in self._items.iteritems():

 

1077

            for prefix, node in viewitems(self._items):

1075

1078

                if node.__class__ is StaticTuple:

1076

1079

                    keys[node] = (prefix, None)

1077

1080

                else:

1147

1150

            else:

1148

1151

                # The slow way. We walk every item in self._items, and check to

1149

1152

                # see if there are any matches

1150

 

                length_filters = length_filters.items()

1151

 

                for prefix, node in self._items.iteritems():

 

1153

                length_filters_itemview = viewitems(length_filters)

 

1154

                for prefix, node in viewitems(self._items):

1152

1155

                    node_key_filter = []

1153

 

                    for length, length_filter in length_filters:

 

1156

                    for length, length_filter in length_filters_itemview:

1154

1157

                        sub_prefix = prefix[:length]

1155

1158

                        if sub_prefix in length_filter:

1156

1159

                            node_key_filter.extend(prefix_to_keys[sub_prefix])

1292

1295

        :param store: A VersionedFiles honouring the CHK extensions.

1293

1296

        :return: An iterable of the keys inserted by this operation.

1294

1297

        """

1295

 

        for node in self._items.itervalues():

 

1298

        for node in viewvalues(self._items):

1296

1299

            if isinstance(node, StaticTuple):

1297

1300

                # Never deserialised.

1298

1301

                continue

1309

1312

            raise AssertionError("_search_prefix should not be None")

1310

1313

        lines.append('%s\n' % (self._search_prefix,))

1311

1314

        prefix_len = len(self._search_prefix)

1312

 

        for prefix, node in sorted(self._items.items()):

 

1315

        for prefix, node in sorted(viewitems(self._items)):

1313

1316

            if isinstance(node, StaticTuple):

1314

1317

                key = node[0]

1315

1318

            else:

1342

1345

            prefix for reaching node.

1343

1346

        """

1344

1347

        if offset >= self._node_width:

1345

 

            for node in self._items.values():

 

1348

            for node in valueview(self._items):

1346

1349

                for result in node._split(offset):

1347

1350

                    yield result

1348

 

            return

1349

 

        for key, node in self._items.items():

1350

 

            pass

1351

1351

 

1352

1352

    def refs(self):

1353

1353

        """Return the references to other CHK's held by this node."""

1354

1354

        if self._key is None:

1355

1355

            raise AssertionError("unserialised nodes have no refs.")

1356

1356

        refs = []

1357

 

        for value in self._items.itervalues():

 

1357

        for value in viewvalues(self._items):

1358

1358

            if isinstance(value, StaticTuple):

1359

1359

                refs.append(value)

1360

1360

            else:

1393

1393

            self._items[search_key] = unmapped

1394

1394

        if len(self._items) == 1:

1395

1395

            # this node is no longer needed:

1396

 

            return self._items.values()[0]

 

1396

            return list(viewvalues(self._items))[0]

1397

1397

        if isinstance(unmapped, InternalNode):

1398

1398

            return self

1399

1399

        if check_remap:

1443

1443

            if isinstance(node, InternalNode):

1444

1444

                # Without looking at any leaf nodes, we are sure

1445

1445

                return self

1446

 

            for key, value in node._items.iteritems():

 

1446

            for key, value in viewitems(node._items):

1447

1447

                if new_leaf._map_no_split(key, value):

1448

1448

                    return self

1449

1449

        trace.mutter("remap generated a new LeafNode")

1532

1532

                #       indicate that we keep 100k prefix_refs around while

1533

1533

                #       processing. They *should* be shorter lived than that...

1534

1534

                #       It does cost us ~10s of processing time

1535

 

                #prefix_refs = [as_st(item) for item in node._items.iteritems()]

1536

 

                prefix_refs = node._items.items()

 

1535

                prefix_refs = list(viewitems(node._items))

1537

1536

                items = []

1538

1537

            else:

1539

1538

                prefix_refs = []

1540

1539

                # Note: We don't use a StaticTuple here. Profiling showed a

1541

1540

                #       minor memory improvement (0.8MB out of 335MB peak 0.2%)

1542

1541

                #       But a significant slowdown (15s / 145s, or 10%)

1543

 

                items = node._items.items()

 

1542

                items = list(viewitems(node._items))

1544

1543

            yield record, node, prefix_refs, items

1545

1544

 

1546

1545

    def _read_old_roots(self):

• Older »

Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0