/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/bzr/chk_map.py

• browse files at revision 7492
• compare with another revision
• download diff
• download tarball
• view history from revision 7492

Show diffs side-by-side

added

removed

breezy/bzr/chk_map.py

50

50

    static_tuple,

51

51

    trace,

52

52

    )

53

 

from ..sixish import (

54

 

    viewitems,

55

 

    viewvalues,

56

 

    )

57

 

from ..sixish import PY3

58

53

from ..static_tuple import StaticTuple

59

54

 

60

55

# approx 4MB

191

186

    def _dump_tree(self, include_keys=False, encoding='utf-8'):

192

187

        """Return the tree in a string representation."""

193

188

        self._ensure_root()

194

 

        if PY3:

195

 

            def decode(x): return x.decode(encoding)

196

 

        else:

197

 

            def decode(x): return x

 

189

        def decode(x): return x.decode(encoding)

198

190

        res = self._dump_tree_node(self._root_node, prefix=b'', indent='',

199

191

                                   decode=decode, include_keys=include_keys)

200

192

        res.append('')  # Give a trailing '\n'

216

208

        if isinstance(node, InternalNode):

217

209

            # Trigger all child nodes to get loaded

218

210

            list(node._iter_nodes(self._store))

219

 

            for prefix, sub in sorted(viewitems(node._items)):

 

211

            for prefix, sub in sorted(node._items.items()):

220

212

                result.extend(self._dump_tree_node(sub, prefix, indent + '  ',

221

213

                                                   decode=decode, include_keys=include_keys))

222

214

        else:

223

 

            for key, value in sorted(viewitems(node._items)):

 

215

            for key, value in sorted(node._items.items()):

224

216

                # Don't use prefix nor indent here to line up when used in

225

217

                # tests in conjunction with assertEqualDiff

226

218

                result.append('      %r %r' % (

260

252

        result._root_node.set_maximum_size(maximum_size)

261

253

        result._root_node._key_width = key_width

262

254

        delta = []

263

 

        for key, value in viewitems(initial_value):

 

255

        for key, value in initial_value.items():

264

256

            delta.append((None, key, value))

265

257

        root_key = result.apply_delta(delta)

266

258

        return root_key

273

265

        node._key_width = key_width

274

266

        as_st = StaticTuple.from_sequence

275

267

        node._items = dict((as_st(key), val)

276

 

                           for key, val in viewitems(initial_value))

 

268

                           for key, val in initial_value.items())

277

269

        node._raw_size = sum(node._key_value_len(key, value)

278

 

                             for key, value in viewitems(node._items))

 

270

                             for key, value in node._items.items())

279

271

        node._len = len(node._items)

280

272

        node._compute_search_prefix()

281

273

        node._compute_serialised_prefix()

339

331

            node = a_map._get_node(node)

340

332

            if isinstance(node, LeafNode):

341

333

                path = (node._key, path)

342

 

                for key, value in viewitems(node._items):

 

334

                for key, value in node._items.items():

343

335

                    # For a LeafNode, the key is a serialized_key, rather than

344

336

                    # a search_key, but the heap is using search_keys

345

337

                    search_key = node._search_key_func(key)

347

339

            else:

348

340

                # type(node) == InternalNode

349

341

                path = (node._key, path)

350

 

                for prefix, child in viewitems(node._items):

 

342

                for prefix, child in node._items.items():

351

343

                    heapq.heappush(pending, (prefix, None, child, path))

352

344

 

353

345

        def process_common_internal_nodes(self_node, basis_node):

354

 

            self_items = set(viewitems(self_node._items))

355

 

            basis_items = set(viewitems(basis_node._items))

 

346

            self_items = set(self_node._items.items())

 

347

            basis_items = set(basis_node._items.items())

356

348

            path = (self_node._key, None)

357

349

            for prefix, child in self_items - basis_items:

358

350

                heapq.heappush(self_pending, (prefix, None, child, path))

361

353

                heapq.heappush(basis_pending, (prefix, None, child, path))

362

354

 

363

355

        def process_common_leaf_nodes(self_node, basis_node):

364

 

            self_items = set(viewitems(self_node._items))

365

 

            basis_items = set(viewitems(basis_node._items))

 

356

            self_items = set(self_node._items.items())

 

357

            basis_items = set(basis_node._items.items())

366

358

            path = (self_node._key, None)

367

359

            for key, value in self_items - basis_items:

368

360

                prefix = self._search_key_func(key)

779

771

                    # Short items, we need to match based on a prefix

780

772

                    filters.setdefault(len(key), set()).add(key)

781

773

            if filters:

782

 

                filters_itemview = viewitems(filters)

783

 

                for item in viewitems(self._items):

 

774

                filters_itemview = filters.items()

 

775

                for item in self._items.items():

784

776

                    for length, length_filter in filters_itemview:

785

777

                        if item[0][:length] in length_filter:

786

778

                            yield item

787

779

                            break

788

780

        else:

789

 

            for item in viewitems(self._items):

790

 

                yield item

 

781

            yield from self._items.items()

791

782

 

792

783

    def _key_value_len(self, key, value):

793

784

        # TODO: Should probably be done without actually joining the key, but

848

839

        common_prefix = self._search_prefix

849

840

        split_at = len(common_prefix) + 1

850

841

        result = {}

851

 

        for key, value in viewitems(self._items):

 

842

        for key, value in self._items.items():

852

843

            search_key = self._search_key(key)

853

844

            prefix = search_key[:split_at]

854

845

            # TODO: Generally only 1 key can be exactly the right length,

881

872

                for split, node in node_details:

882

873

                    new_node.add_node(split, node)

883

874

                result[prefix] = new_node

884

 

        return common_prefix, list(viewitems(result))

 

875

        return common_prefix, list(result.items())

885

876

 

886

877

    def map(self, store, key, value):

887

878

        """Map key to value."""

916

907

        else:

917

908

            lines.append(b'%s\n' % (self._common_serialised_prefix,))

918

909

            prefix_len = len(self._common_serialised_prefix)

919

 

        for key, value in sorted(viewitems(self._items)):

 

910

        for key, value in sorted(self._items.items()):

920

911

            # Always add a final newline

921

912

            value_lines = osutils.chunks_to_lines([value + b'\n'])

922

913

            serialized = b"%s\x00%d\n" % (self._serialise_key(key),

1081

1072

            # yielding all nodes, yield whatever we have, and queue up a read

1082

1073

            # for whatever we are missing

1083

1074

            shortcut = True

1084

 

            for prefix, node in viewitems(self._items):

 

1075

            for prefix, node in self._items.items():

1085

1076

                if node.__class__ is StaticTuple:

1086

1077

                    keys[node] = (prefix, None)

1087

1078

                else:

1157

1148

            else:

1158

1149

                # The slow way. We walk every item in self._items, and check to

1159

1150

                # see if there are any matches

1160

 

                length_filters_itemview = viewitems(length_filters)

1161

 

                for prefix, node in viewitems(self._items):

 

1151

                length_filters_itemview = length_filters.items()

 

1152

                for prefix, node in self._items.items():

1162

1153

                    node_key_filter = []

1163

1154

                    for length, length_filter in length_filters_itemview:

1164

1155

                        sub_prefix = prefix[:length]

1302

1293

        :param store: A VersionedFiles honouring the CHK extensions.

1303

1294

        :return: An iterable of the keys inserted by this operation.

1304

1295

        """

1305

 

        for node in viewvalues(self._items):

 

1296

        for node in self._items.values():

1306

1297

            if isinstance(node, StaticTuple):

1307

1298

                # Never deserialised.

1308

1299

                continue

1319

1310

            raise AssertionError("_search_prefix should not be None")

1320

1311

        lines.append(b'%s\n' % (self._search_prefix,))

1321

1312

        prefix_len = len(self._search_prefix)

1322

 

        for prefix, node in sorted(viewitems(self._items)):

 

1313

        for prefix, node in sorted(self._items.items()):

1323

1314

            if isinstance(node, StaticTuple):

1324

1315

                key = node[0]

1325

1316

            else:

1361

1352

        if self._key is None:

1362

1353

            raise AssertionError("unserialised nodes have no refs.")

1363

1354

        refs = []

1364

 

        for value in viewvalues(self._items):

 

1355

        for value in self._items.values():

1365

1356

            if isinstance(value, StaticTuple):

1366

1357

                refs.append(value)

1367

1358

            else:

1400

1391

            self._items[search_key] = unmapped

1401

1392

        if len(self._items) == 1:

1402

1393

            # this node is no longer needed:

1403

 

            return list(viewvalues(self._items))[0]

 

1394

            return list(self._items.values())[0]

1404

1395

        if isinstance(unmapped, InternalNode):

1405

1396

            return self

1406

1397

        if check_remap:

1450

1441

            if isinstance(node, InternalNode):

1451

1442

                # Without looking at any leaf nodes, we are sure

1452

1443

                return self

1453

 

            for key, value in viewitems(node._items):

 

1444

            for key, value in node._items.items():

1454

1445

                if new_leaf._map_no_split(key, value):

1455

1446

                    return self

1456

1447

        trace.mutter("remap generated a new LeafNode")

1539

1530

                #       indicate that we keep 100k prefix_refs around while

1540

1531

                #       processing. They *should* be shorter lived than that...

1541

1532

                #       It does cost us ~10s of processing time

1542

 

                prefix_refs = list(viewitems(node._items))

 

1533

                prefix_refs = list(node._items.items())

1543

1534

                items = []

1544

1535

            else:

1545

1536

                prefix_refs = []

1546

1537

                # Note: We don't use a StaticTuple here. Profiling showed a

1547

1538

                #       minor memory improvement (0.8MB out of 335MB peak 0.2%)

1548

1539

                #       But a significant slowdown (15s / 145s, or 10%)

1549

 

                items = list(viewitems(node._items))

 

1540

                items = list(node._items.items())

1550

1541

            yield record, node, prefix_refs, items

1551

1542

 

1552

1543

    def _read_old_roots(self):

• Older »

Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0