/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

• browse files at revision 5757
• compare with another revision
• download diff
• download tarball
• view history from revision 5757

• reverse the comparison (5757 to 5200.3.2)
• stop comparing with revision 5200.3.2

Show diffs side-by-side

added

removed

bzrlib/repofmt/groupcompress_repo.py

1

 

# Copyright (C) 2008, 2009, 2010 Canonical Ltd

 

1

# Copyright (C) 2008-2011 Canonical Ltd

2

2

#

3

3

# This program is free software; you can redistribute it and/or modify

4

4

# it under the terms of the GNU General Public License as published by

32

32

    revision as _mod_revision,

33

33

    trace,

34

34

    ui,

 

35

    versionedfile,

35

36

    )

36

37

from bzrlib.btree_index import (

37

38

    BTreeGraphIndex,

38

39

    BTreeBuilder,

39

40

    )

 

41

from bzrlib.decorators import needs_write_lock

40

42

from bzrlib.groupcompress import (

41

43

    _GCGraphIndex,

42

44

    GroupCompressVersionedFiles,

262

264

        remaining_keys = set(keys)

263

265

        counter = [0]

264

266

        if self._gather_text_refs:

265

 

            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

266

267

            self._text_refs = set()

267

268

        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):

268

269

            cur_keys = root_keys

289

290

                    # Store is None, because we know we have a LeafNode, and we

290

291

                    # just want its entries

291

292

                    for file_id, bytes in node.iteritems(None):

292

 

                        name_utf8, file_id, revision_id = bytes_to_info(bytes)

293

 

                        self._text_refs.add((file_id, revision_id))

 

293

                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))

294

294

                def next_stream():

295

295

                    stream = source_vf.get_record_stream(cur_keys,

296

296

                                                         'as-requested', True)

421

421

        inventory_keys = source_vf.keys()

422

422

        missing_inventories = set(self.revision_keys).difference(inventory_keys)

423

423

        if missing_inventories:

424

 

            missing_inventories = sorted(missing_inventories)

425

 

            raise ValueError('We are missing inventories for revisions: %s'

426

 

                % (missing_inventories,))

 

424

            # Go back to the original repo, to see if these are really missing

 

425

            # https://bugs.launchpad.net/bzr/+bug/437003

 

426

            # If we are packing a subset of the repo, it is fine to just have

 

427

            # the data in another Pack file, which is not included in this pack

 

428

            # operation.

 

429

            inv_index = self._pack_collection.repo.inventories._index

 

430

            pmap = inv_index.get_parent_map(missing_inventories)

 

431

            really_missing = missing_inventories.difference(pmap)

 

432

            if really_missing:

 

433

                missing_inventories = sorted(really_missing)

 

434

                raise ValueError('We are missing inventories for revisions: %s'

 

435

                    % (missing_inventories,))

427

436

        self._copy_stream(source_vf, target_vf, inventory_keys,

428

437

                          'inventories', self._get_filtered_inv_stream, 2)

429

438

 

 

439

    def _get_chk_vfs_for_copy(self):

 

440

        return self._build_vfs('chk', False, False)

 

441

 

430

442

    def _copy_chk_texts(self):

431

 

        source_vf, target_vf = self._build_vfs('chk', False, False)

 

443

        source_vf, target_vf = self._get_chk_vfs_for_copy()

432

444

        # TODO: This is technically spurious... if it is a performance issue,

433

445

        #       remove it

434

446

        total_keys = source_vf.keys()

580

592

        return new_pack.data_inserted() and self._data_changed

581

593

 

582

594

 

 

595

class GCCHKCanonicalizingPacker(GCCHKPacker):

 

596

    """A packer that ensures inventories have canonical-form CHK maps.

 

597

    

 

598

    Ideally this would be part of reconcile, but it's very slow and rarely

 

599

    needed.  (It repairs repositories affected by

 

600

    https://bugs.launchpad.net/bzr/+bug/522637).

 

601

    """

 

602

 

 

603

    def __init__(self, *args, **kwargs):

 

604

        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)

 

605

        self._data_changed = False

 

606

    

 

607

    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):

 

608

        """Create and exhaust a stream, but don't insert it.

 

609

        

 

610

        This is useful to get the side-effects of generating a stream.

 

611

        """

 

612

        self.pb.update('scanning %s' % (message,), pb_offset)

 

613

        child_pb = ui.ui_factory.nested_progress_bar()

 

614

        try:

 

615

            list(vf_to_stream(source_vf, keys, message, child_pb))

 

616

        finally:

 

617

            child_pb.finished()

 

618

 

 

619

    def _copy_inventory_texts(self):

 

620

        source_vf, target_vf = self._build_vfs('inventory', True, True)

 

621

        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()

 

622

        inventory_keys = source_vf.keys()

 

623

        # First, copy the existing CHKs on the assumption that most of them

 

624

        # will be correct.  This will save us from having to reinsert (and

 

625

        # recompress) these records later at the cost of perhaps preserving a

 

626

        # few unused CHKs. 

 

627

        # (Iterate but don't insert _get_filtered_inv_stream to populate the

 

628

        # variables needed by GCCHKPacker._copy_chk_texts.)

 

629

        self._exhaust_stream(source_vf, inventory_keys, 'inventories',

 

630

                self._get_filtered_inv_stream, 2)

 

631

        GCCHKPacker._copy_chk_texts(self)

 

632

        # Now copy and fix the inventories, and any regenerated CHKs.

 

633

        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):

 

634

            return self._get_filtered_canonicalizing_inv_stream(

 

635

                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)

 

636

        self._copy_stream(source_vf, target_vf, inventory_keys,

 

637

                          'inventories', chk_canonicalizing_inv_stream, 4)

 

638

 

 

639

    def _copy_chk_texts(self):

 

640

        # No-op; in this class this happens during _copy_inventory_texts.

 

641

        pass

 

642

 

 

643

    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,

 

644

            pb=None, source_chk_vf=None, target_chk_vf=None):

 

645

        """Filter the texts of inventories, regenerating CHKs to make sure they

 

646

        are canonical.

 

647

        """

 

648

        total_keys = len(keys)

 

649

        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)

 

650

        def _filtered_inv_stream():

 

651

            stream = source_vf.get_record_stream(keys, 'groupcompress', True)

 

652

            search_key_name = None

 

653

            for idx, record in enumerate(stream):

 

654

                # Inventories should always be with revisions; assume success.

 

655

                bytes = record.get_bytes_as('fulltext')

 

656

                chk_inv = inventory.CHKInventory.deserialise(

 

657

                    source_chk_vf, bytes, record.key)

 

658

                if pb is not None:

 

659

                    pb.update('inv', idx, total_keys)

 

660

                chk_inv.id_to_entry._ensure_root()

 

661

                if search_key_name is None:

 

662

                    # Find the name corresponding to the search_key_func

 

663

                    search_key_reg = chk_map.search_key_registry

 

664

                    for search_key_name, func in search_key_reg.iteritems():

 

665

                        if func == chk_inv.id_to_entry._search_key_func:

 

666

                            break

 

667

                canonical_inv = inventory.CHKInventory.from_inventory(

 

668

                    target_chk_vf, chk_inv,

 

669

                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,

 

670

                    search_key_name=search_key_name)

 

671

                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():

 

672

                    trace.mutter(

 

673

                        'Non-canonical CHK map for id_to_entry of inv: %s '

 

674

                        '(root is %s, should be %s)' % (chk_inv.revision_id,

 

675

                        chk_inv.id_to_entry.key()[0],

 

676

                        canonical_inv.id_to_entry.key()[0]))

 

677

                    self._data_changed = True

 

678

                p_id_map = chk_inv.parent_id_basename_to_file_id

 

679

                p_id_map._ensure_root()

 

680

                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id

 

681

                if p_id_map.key() != canon_p_id_map.key():

 

682

                    trace.mutter(

 

683

                        'Non-canonical CHK map for parent_id_to_basename of '

 

684

                        'inv: %s (root is %s, should be %s)'

 

685

                        % (chk_inv.revision_id, p_id_map.key()[0],

 

686

                           canon_p_id_map.key()[0]))

 

687

                    self._data_changed = True

 

688

                yield versionedfile.ChunkedContentFactory(record.key,

 

689

                        record.parents, record.sha1,

 

690

                        canonical_inv.to_lines())

 

691

            # We have finished processing all of the inventory records, we

 

692

            # don't need these sets anymore

 

693

        return _filtered_inv_stream()

 

694

 

 

695

    def _use_pack(self, new_pack):

 

696

        """Override _use_pack to check for reconcile having changed content."""

 

697

        return new_pack.data_inserted() and self._data_changed

 

698

 

 

699

 

583

700

class GCRepositoryPackCollection(RepositoryPackCollection):

584

701

 

585

702

    pack_factory = GCPack

647

764

        chk_diff = chk_map.iter_interesting_nodes(

648

765

            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,

649

766

            root_key_info.uninteresting_root_keys)

650

 

        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

651

767

        text_keys = set()

652

768

        try:

653

 

            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):

 

769

            for record in _filter_text_keys(chk_diff, text_keys,

 

770

                                            chk_map._bytes_to_text_key):

654

771

                pass

655

772

        except errors.NoSuchRevision, e:

656

773

            # XXX: It would be nice if we could give a more precise error here.

864

981

        if basis_inv is None:

865

982

            if basis_revision_id == _mod_revision.NULL_REVISION:

866

983

                new_inv = self._create_inv_from_null(delta, new_revision_id)

 

984

                if new_inv.root_id is None:

 

985

                    raise errors.RootMissing()

867

986

                inv_lines = new_inv.to_lines()

868

987

                return self._inventory_add_lines(new_revision_id, parents,

869

988

                    inv_lines, check_content=False), new_inv

999

1118

        finally:

1000

1119

            pb.finished()

1001

1120

 

 

1121

    @needs_write_lock

 

1122

    def reconcile_canonicalize_chks(self):

 

1123

        """Reconcile this repository to make sure all CHKs are in canonical

 

1124

        form.

 

1125

        """

 

1126

        from bzrlib.reconcile import PackReconciler

 

1127

        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)

 

1128

        reconciler.reconcile()

 

1129

        return reconciler

 

1130

 

1002

1131

    def _reconcile_pack(self, collection, packs, extension, revs, pb):

1003

1132

        packer = GCCHKReconcilePacker(collection, packs, extension)

1004

1133

        return packer.pack(pb)

1005

1134

 

 

1135

    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):

 

1136

        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)

 

1137

        return packer.pack(pb)

 

1138

 

1006

1139

    def _get_source(self, to_format):

1007

1140

        """Return a source for streaming from this repository."""

1008

1141

        if self._format._serializer == to_format._serializer:

1087

1220

                uninteresting_root_keys.add(inv.id_to_entry.key())

1088

1221

                uninteresting_pid_root_keys.add(

1089

1222

                    inv.parent_id_basename_to_file_id.key())

1090

 

        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key

1091

1223

        chk_bytes = self.from_repository.chk_bytes

1092

1224

        def _filter_id_to_entry():

1093

1225

            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,

1094

1226

                        self._chk_id_roots, uninteresting_root_keys)

1095

1227

            for record in _filter_text_keys(interesting_nodes, self._text_keys,

1096

 

                    bytes_to_info):

 

1228

                    chk_map._bytes_to_text_key):

1097

1229

                if record is not None:

1098

1230

                    yield record

1099

1231

            # Consumed

1109

1241

        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()

1110

1242

 

1111

1243

    def get_stream(self, search):

 

1244

        def wrap_and_count(pb, rc, stream):

 

1245

            """Yield records from stream while showing progress."""

 

1246

            count = 0

 

1247

            for record in stream:

 

1248

                if count == rc.STEP:

 

1249

                    rc.increment(count)

 

1250

                    pb.update('Estimate', rc.current, rc.max)

 

1251

                    count = 0

 

1252

                count += 1

 

1253

                yield record

 

1254

 

1112

1255

        revision_ids = search.get_keys()

 

1256

        pb = ui.ui_factory.nested_progress_bar()

 

1257

        rc = self._record_counter

 

1258

        self._record_counter.setup(len(revision_ids))

1113

1259

        for stream_info in self._fetch_revision_texts(revision_ids):

1114

 

            yield stream_info

 

1260

            yield (stream_info[0],

 

1261

                wrap_and_count(pb, rc, stream_info[1]))

1115

1262

        self._revision_keys = [(rev_id,) for rev_id in revision_ids]

1116

1263

        self.from_repository.revisions.clear_cache()

1117

1264

        self.from_repository.signatures.clear_cache()

1118

 

        yield self._get_inventory_stream(self._revision_keys)

 

1265

        s = self._get_inventory_stream(self._revision_keys)

 

1266

        yield (s[0], wrap_and_count(pb, rc, s[1]))

1119

1267

        self.from_repository.inventories.clear_cache()

1120

1268

        # TODO: The keys to exclude might be part of the search recipe

1121

1269

        # For now, exclude all parents that are at the edge of ancestry, for

1124

1272

        parent_keys = from_repo._find_parent_keys_of_revisions(

1125

1273

                        self._revision_keys)

1126

1274

        for stream_info in self._get_filtered_chk_streams(parent_keys):

1127

 

            yield stream_info

 

1275

            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))

1128

1276

        self.from_repository.chk_bytes.clear_cache()

1129

 

        yield self._get_text_stream()

 

1277

        s = self._get_text_stream()

 

1278

        yield (s[0], wrap_and_count(pb, rc, s[1]))

1130

1279

        self.from_repository.texts.clear_cache()

 

1280

        pb.update('Done', rc.max, rc.max)

 

1281

        pb.finished()

1131

1282

 

1132

1283

    def get_stream_for_missing_keys(self, missing_keys):

1133

1284

        # missing keys can only occur when we are byte copying and not

1187

1338

    return result

1188

1339

 

1189

1340

 

1190

 

def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):

 

1341

def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):

1191

1342

    """Iterate the result of iter_interesting_nodes, yielding the records

1192

1343

    and adding to text_keys.

1193

1344

    """

 

1345

    text_keys_update = text_keys.update

1194

1346

    for record, items in interesting_nodes_iterable:

1195

 

        for name, bytes in items:

1196

 

            # Note: we don't care about name_utf8, because groupcompress repos

1197

 

            # are always rich-root, so there are no synthesised root records to

1198

 

            # ignore.

1199

 

            _, file_id, revision_id = bytes_to_info(bytes)

1200

 

            file_id = intern(file_id)

1201

 

            revision_id = intern(revision_id)

1202

 

            text_keys.add(StaticTuple(file_id, revision_id).intern())

 

1347

        text_keys_update([bytes_to_text_key(b) for n,b in items])

1203

1348

        yield record

1204

1349

 

1205

1350

 

1206

 

 

1207

 

 

1208

 

class RepositoryFormatCHK1(RepositoryFormatPack):

1209

 

    """A hashed CHK+group compress pack repository."""

 

1351

class RepositoryFormat2a(RepositoryFormatPack):

 

1352

    """A CHK repository that uses the bencode revision serializer."""

1210

1353

 

1211

1354

    repository_class = CHKInventoryRepository

1212

1355

    supports_external_lookups = True

1213

1356

    supports_chks = True

1214

 

    # For right now, setting this to True gives us InterModel1And2 rather

1215

 

    # than InterDifferingSerializer

1216

1357

    _commit_builder_class = PackRootCommitBuilder

1217

1358

    rich_root_data = True

1218

 

    _serializer = chk_serializer.chk_serializer_255_bigpage

 

1359

    _serializer = chk_serializer.chk_bencode_serializer

1219

1360

    _commit_inv_deltas = True

1220

1361

    # What index classes to use

1221

1362

    index_builder_class = BTreeBuilder

1232

1373

    pack_compresses = True

1233

1374

 

1234

1375

    def _get_matching_bzrdir(self):

1235

 

        return bzrdir.format_registry.make_bzrdir('development6-rich-root')

1236

 

 

1237

 

    def _ignore_setting_bzrdir(self, format):

1238

 

        pass

1239

 

 

1240

 

    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)

1241

 

 

1242

 

    def get_format_string(self):

1243

 

        """See RepositoryFormat.get_format_string()."""

1244

 

        return ('Bazaar development format - group compression and chk inventory'

1245

 

                ' (needs bzr.dev from 1.14)\n')

1246

 

 

1247

 

    def get_format_description(self):

1248

 

        """See RepositoryFormat.get_format_description()."""

1249

 

        return ("Development repository format - rich roots, group compression"

1250

 

            " and chk inventories")

1251

 

 

1252

 

 

1253

 

class RepositoryFormatCHK2(RepositoryFormatCHK1):

1254

 

    """A CHK repository that uses the bencode revision serializer."""

1255

 

 

1256

 

    _serializer = chk_serializer.chk_bencode_serializer

1257

 

 

1258

 

    def _get_matching_bzrdir(self):

1259

 

        return bzrdir.format_registry.make_bzrdir('development7-rich-root')

1260

 

 

1261

 

    def _ignore_setting_bzrdir(self, format):

1262

 

        pass

1263

 

 

1264

 

    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)

1265

 

 

1266

 

    def get_format_string(self):

1267

 

        """See RepositoryFormat.get_format_string()."""

1268

 

        return ('Bazaar development format - chk repository with bencode '

1269

 

                'revision serialization (needs bzr.dev from 1.16)\n')

1270

 

 

1271

 

 

1272

 

class RepositoryFormat2a(RepositoryFormatCHK2):

1273

 

    """A CHK repository that uses the bencode revision serializer.

1274

 

 

1275

 

    This is the same as RepositoryFormatCHK2 but with a public name.

1276

 

    """

1277

 

 

1278

 

    _serializer = chk_serializer.chk_bencode_serializer

1279

 

 

1280

 

    def _get_matching_bzrdir(self):

1281

1376

        return bzrdir.format_registry.make_bzrdir('2a')

1282

1377

 

1283

1378

    def _ignore_setting_bzrdir(self, format):

1292

1387

        """See RepositoryFormat.get_format_description()."""

1293

1388

        return ("Repository format 2a - rich roots, group compression"

1294

1389

            " and chk inventories")

 

1390

 

 

1391

 

 

1392

class RepositoryFormat2aSubtree(RepositoryFormat2a):

 

1393

    """A 2a repository format that supports nested trees.

 

1394

 

 

1395

    """

 

1396

 

 

1397

    def _get_matching_bzrdir(self):

 

1398

        return bzrdir.format_registry.make_bzrdir('development-subtree')

 

1399

 

 

1400

    def _ignore_setting_bzrdir(self, format):

 

1401

        pass

 

1402

 

 

1403

    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)

 

1404

 

 

1405

    def get_format_string(self):

 

1406

        return ('Bazaar development format 8\n')

 

1407

 

 

1408

    def get_format_description(self):

 

1409

        """See RepositoryFormat.get_format_description()."""

 

1410

        return ("Development repository format 8 - nested trees, "

 

1411

                "group compression and chk inventories")

 

1412

 

 

1413

    experimental = True

 

1414

    supports_tree_reference = True

• Older »

Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0