/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to brzlib/repofmt/knitpack_repo.py

  • Committer: Jelmer Vernooij
  • Date: 2017-05-21 12:41:27 UTC
  • mto: This revision was merged to the branch mainline in revision 6623.
  • Revision ID: jelmer@jelmer.uk-20170521124127-iv8etg0vwymyai6y
s/bzr/brz/ in apport config.

Show diffs side-by-side

added added

removed removed

Lines of Context:
16
16
 
17
17
"""Knit-based pack repository formats."""
18
18
 
19
 
from .. import (
20
 
    errors,
21
 
    )
 
19
from __future__ import absolute_import
22
20
 
23
 
from ..lazy_import import lazy_import
 
21
from brzlib.lazy_import import lazy_import
24
22
lazy_import(globals(), """
 
23
from itertools import izip
25
24
import time
26
25
 
27
 
from breezy import (
 
26
from brzlib import (
28
27
    controldir,
29
28
    debug,
 
29
    errors,
 
30
    knit,
30
31
    osutils,
 
32
    pack,
31
33
    revision as _mod_revision,
32
34
    trace,
33
35
    tsort,
34
36
    ui,
35
 
    )
36
 
from breezy.bzr import (
37
 
    pack,
38
37
    xml5,
39
38
    xml6,
40
39
    xml7,
41
40
    )
42
 
from breezy.bzr.knit import (
 
41
from brzlib.knit import (
43
42
    _KnitGraphIndex,
44
43
    KnitPlainFactory,
45
44
    KnitVersionedFiles,
46
45
    )
47
46
""")
48
47
 
49
 
from ..bzr import (
 
48
from brzlib import (
50
49
    btree_index,
51
50
    )
52
 
from ..bzr.index import (
 
51
from brzlib.index import (
53
52
    CombinedGraphIndex,
54
53
    GraphIndex,
55
54
    GraphIndexPrefixAdapter,
56
55
    InMemoryGraphIndex,
57
56
    )
58
 
from .knitrepo import (
 
57
from brzlib.repofmt.knitrepo import (
59
58
    KnitRepository,
60
59
    )
61
 
from .pack_repo import (
 
60
from brzlib.repofmt.pack_repo import (
62
61
    _DirectPackAccess,
63
62
    NewPack,
64
63
    RepositoryFormatPack,
66
65
    Packer,
67
66
    PackCommitBuilder,
68
67
    PackRepository,
 
68
    PackRootCommitBuilder,
69
69
    RepositoryPackCollection,
70
70
    )
71
 
from ..bzr.vf_repository import (
 
71
from brzlib.vf_repository import (
72
72
    StreamSource,
73
73
    )
74
74
 
75
75
 
76
76
class KnitPackRepository(PackRepository, KnitRepository):
77
77
 
78
 
    def __init__(self, _format, a_controldir, control_files, _commit_builder_class,
79
 
                 _serializer):
80
 
        PackRepository.__init__(self, _format, a_controldir, control_files,
81
 
                                _commit_builder_class, _serializer)
 
78
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
 
79
        _serializer):
 
80
        PackRepository.__init__(self, _format, a_bzrdir, control_files,
 
81
            _commit_builder_class, _serializer)
82
82
        if self._format.supports_chks:
83
83
            raise AssertionError("chk not supported")
84
84
        index_transport = self._transport.clone('indices')
85
85
        self._pack_collection = KnitRepositoryPackCollection(self,
86
 
                                                             self._transport,
87
 
                                                             index_transport,
88
 
                                                             self._transport.clone(
89
 
                                                                 'upload'),
90
 
                                                             self._transport.clone(
91
 
                                                                 'packs'),
92
 
                                                             _format.index_builder_class,
93
 
                                                             _format.index_class,
94
 
                                                             use_chk_index=False,
95
 
                                                             )
 
86
            self._transport,
 
87
            index_transport,
 
88
            self._transport.clone('upload'),
 
89
            self._transport.clone('packs'),
 
90
            _format.index_builder_class,
 
91
            _format.index_class,
 
92
            use_chk_index=False,
 
93
            )
96
94
        self.inventories = KnitVersionedFiles(
97
95
            _KnitGraphIndex(self._pack_collection.inventory_index.combined_index,
98
 
                            add_callback=self._pack_collection.inventory_index.add_callback,
99
 
                            deltas=True, parents=True, is_locked=self.is_locked),
 
96
                add_callback=self._pack_collection.inventory_index.add_callback,
 
97
                deltas=True, parents=True, is_locked=self.is_locked),
100
98
            data_access=self._pack_collection.inventory_index.data_access,
101
99
            max_delta_chain=200)
102
100
        self.revisions = KnitVersionedFiles(
103
101
            _KnitGraphIndex(self._pack_collection.revision_index.combined_index,
104
 
                            add_callback=self._pack_collection.revision_index.add_callback,
105
 
                            deltas=False, parents=True, is_locked=self.is_locked,
106
 
                            track_external_parent_refs=True),
 
102
                add_callback=self._pack_collection.revision_index.add_callback,
 
103
                deltas=False, parents=True, is_locked=self.is_locked,
 
104
                track_external_parent_refs=True),
107
105
            data_access=self._pack_collection.revision_index.data_access,
108
106
            max_delta_chain=0)
109
107
        self.signatures = KnitVersionedFiles(
110
108
            _KnitGraphIndex(self._pack_collection.signature_index.combined_index,
111
 
                            add_callback=self._pack_collection.signature_index.add_callback,
112
 
                            deltas=False, parents=False, is_locked=self.is_locked),
 
109
                add_callback=self._pack_collection.signature_index.add_callback,
 
110
                deltas=False, parents=False, is_locked=self.is_locked),
113
111
            data_access=self._pack_collection.signature_index.data_access,
114
112
            max_delta_chain=0)
115
113
        self.texts = KnitVersionedFiles(
116
114
            _KnitGraphIndex(self._pack_collection.text_index.combined_index,
117
 
                            add_callback=self._pack_collection.text_index.add_callback,
118
 
                            deltas=True, parents=True, is_locked=self.is_locked),
 
115
                add_callback=self._pack_collection.text_index.add_callback,
 
116
                deltas=True, parents=True, is_locked=self.is_locked),
119
117
            data_access=self._pack_collection.text_index.data_access,
120
118
            max_delta_chain=200)
121
119
        self.chk_bytes = None
149
147
 
150
148
    repository_class = KnitPackRepository
151
149
    _commit_builder_class = PackCommitBuilder
152
 
 
153
150
    @property
154
151
    def _serializer(self):
155
152
        return xml5.serializer_v5
158
155
    index_class = GraphIndex
159
156
 
160
157
    def _get_matching_bzrdir(self):
161
 
        return controldir.format_registry.make_controldir('pack-0.92')
 
158
        return controldir.format_registry.make_bzrdir('pack-0.92')
162
159
 
163
160
    def _ignore_setting_bzrdir(self, format):
164
161
        pass
165
162
 
166
 
    _matchingcontroldir = property(
167
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
163
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
168
164
 
169
165
    @classmethod
170
166
    def get_format_string(cls):
171
167
        """See RepositoryFormat.get_format_string()."""
172
 
        return b"Bazaar pack repository format 1 (needs bzr 0.92)\n"
 
168
        return "Bazaar pack repository format 1 (needs bzr 0.92)\n"
173
169
 
174
170
    def get_format_description(self):
175
171
        """See RepositoryFormat.get_format_description()."""
187
183
    """
188
184
 
189
185
    repository_class = KnitPackRepository
190
 
    _commit_builder_class = PackCommitBuilder
 
186
    _commit_builder_class = PackRootCommitBuilder
191
187
    rich_root_data = True
192
188
    experimental = True
193
189
    supports_tree_reference = True
194
 
 
195
190
    @property
196
191
    def _serializer(self):
197
192
        return xml7.serializer_v7
200
195
    index_class = GraphIndex
201
196
 
202
197
    def _get_matching_bzrdir(self):
203
 
        return controldir.format_registry.make_controldir(
 
198
        return controldir.format_registry.make_bzrdir(
204
199
            'pack-0.92-subtree')
205
200
 
206
201
    def _ignore_setting_bzrdir(self, format):
207
202
        pass
208
203
 
209
 
    _matchingcontroldir = property(
210
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
204
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
211
205
 
212
206
    @classmethod
213
207
    def get_format_string(cls):
214
208
        """See RepositoryFormat.get_format_string()."""
215
 
        return b"Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
 
209
        return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
216
210
 
217
211
    def get_format_description(self):
218
212
        """See RepositoryFormat.get_format_description()."""
229
223
    """
230
224
 
231
225
    repository_class = KnitPackRepository
232
 
    _commit_builder_class = PackCommitBuilder
 
226
    _commit_builder_class = PackRootCommitBuilder
233
227
    rich_root_data = True
234
228
    supports_tree_reference = False
235
 
 
236
229
    @property
237
230
    def _serializer(self):
238
231
        return xml6.serializer_v6
241
234
    index_class = GraphIndex
242
235
 
243
236
    def _get_matching_bzrdir(self):
244
 
        return controldir.format_registry.make_controldir(
 
237
        return controldir.format_registry.make_bzrdir(
245
238
            'rich-root-pack')
246
239
 
247
240
    def _ignore_setting_bzrdir(self, format):
248
241
        pass
249
242
 
250
 
    _matchingcontroldir = property(
251
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
243
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
252
244
 
253
245
    @classmethod
254
246
    def get_format_string(cls):
255
247
        """See RepositoryFormat.get_format_string()."""
256
 
        return (b"Bazaar pack repository format 1 with rich root"
257
 
                b" (needs bzr 1.0)\n")
 
248
        return ("Bazaar pack repository format 1 with rich root"
 
249
                " (needs bzr 1.0)\n")
258
250
 
259
251
    def get_format_description(self):
260
252
        """See RepositoryFormat.get_format_description()."""
282
274
        return xml5.serializer_v5
283
275
 
284
276
    def _get_matching_bzrdir(self):
285
 
        return controldir.format_registry.make_controldir('1.6')
 
277
        return controldir.format_registry.make_bzrdir('1.6')
286
278
 
287
279
    def _ignore_setting_bzrdir(self, format):
288
280
        pass
289
281
 
290
 
    _matchingcontroldir = property(
291
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
282
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
292
283
 
293
284
    @classmethod
294
285
    def get_format_string(cls):
295
286
        """See RepositoryFormat.get_format_string()."""
296
 
        return b"Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n"
 
287
        return "Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n"
297
288
 
298
289
    def get_format_description(self):
299
290
        """See RepositoryFormat.get_format_description()."""
310
301
    """
311
302
 
312
303
    repository_class = KnitPackRepository
313
 
    _commit_builder_class = PackCommitBuilder
 
304
    _commit_builder_class = PackRootCommitBuilder
314
305
    rich_root_data = True
315
 
    supports_tree_reference = False  # no subtrees
 
306
    supports_tree_reference = False # no subtrees
316
307
    supports_external_lookups = True
317
308
    # What index classes to use
318
309
    index_builder_class = InMemoryGraphIndex
323
314
        return xml6.serializer_v6
324
315
 
325
316
    def _get_matching_bzrdir(self):
326
 
        return controldir.format_registry.make_controldir(
 
317
        return controldir.format_registry.make_bzrdir(
327
318
            '1.6.1-rich-root')
328
319
 
329
320
    def _ignore_setting_bzrdir(self, format):
330
321
        pass
331
322
 
332
 
    _matchingcontroldir = property(
333
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
323
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
334
324
 
335
325
    @classmethod
336
326
    def get_format_string(cls):
337
327
        """See RepositoryFormat.get_format_string()."""
338
 
        return b"Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n"
 
328
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n"
339
329
 
340
330
    def get_format_description(self):
341
331
        return "Packs 5 rich-root (adds stacking support, requires bzr 1.6.1)"
355
345
    """
356
346
 
357
347
    repository_class = KnitPackRepository
358
 
    _commit_builder_class = PackCommitBuilder
 
348
    _commit_builder_class = PackRootCommitBuilder
359
349
    rich_root_data = True
360
 
    supports_tree_reference = False  # no subtrees
 
350
    supports_tree_reference = False # no subtrees
361
351
 
362
352
    supports_external_lookups = True
363
353
    # What index classes to use
369
359
        return xml7.serializer_v7
370
360
 
371
361
    def _get_matching_bzrdir(self):
372
 
        matching = controldir.format_registry.make_controldir(
 
362
        matching = controldir.format_registry.make_bzrdir(
373
363
            '1.6.1-rich-root')
374
364
        matching.repository_format = self
375
365
        return matching
377
367
    def _ignore_setting_bzrdir(self, format):
378
368
        pass
379
369
 
380
 
    _matchingcontroldir = property(
381
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
370
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
382
371
 
383
372
    @classmethod
384
373
    def get_format_string(cls):
385
374
        """See RepositoryFormat.get_format_string()."""
386
 
        return b"Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n"
 
375
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n"
387
376
 
388
377
    def get_format_description(self):
389
378
        return ("Packs 5 rich-root (adds stacking support, requires bzr 1.6)"
412
401
        return xml5.serializer_v5
413
402
 
414
403
    def _get_matching_bzrdir(self):
415
 
        return controldir.format_registry.make_controldir('1.9')
 
404
        return controldir.format_registry.make_bzrdir('1.9')
416
405
 
417
406
    def _ignore_setting_bzrdir(self, format):
418
407
        pass
419
408
 
420
 
    _matchingcontroldir = property(
421
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
409
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
422
410
 
423
411
    @classmethod
424
412
    def get_format_string(cls):
425
413
        """See RepositoryFormat.get_format_string()."""
426
 
        return b"Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n"
 
414
        return "Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n"
427
415
 
428
416
    def get_format_description(self):
429
417
        """See RepositoryFormat.get_format_description()."""
437
425
    """
438
426
 
439
427
    repository_class = KnitPackRepository
440
 
    _commit_builder_class = PackCommitBuilder
 
428
    _commit_builder_class = PackRootCommitBuilder
441
429
    rich_root_data = True
442
 
    supports_tree_reference = False  # no subtrees
 
430
    supports_tree_reference = False # no subtrees
443
431
    supports_external_lookups = True
444
432
    # What index classes to use
445
433
    index_builder_class = btree_index.BTreeBuilder
450
438
        return xml6.serializer_v6
451
439
 
452
440
    def _get_matching_bzrdir(self):
453
 
        return controldir.format_registry.make_controldir(
 
441
        return controldir.format_registry.make_bzrdir(
454
442
            '1.9-rich-root')
455
443
 
456
444
    def _ignore_setting_bzrdir(self, format):
457
445
        pass
458
446
 
459
 
    _matchingcontroldir = property(
460
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
447
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
461
448
 
462
449
    @classmethod
463
450
    def get_format_string(cls):
464
451
        """See RepositoryFormat.get_format_string()."""
465
 
        return b"Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
 
452
        return "Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
466
453
 
467
454
    def get_format_description(self):
468
455
        return "Packs 6 rich-root (uses btree indexes, requires bzr 1.9)"
478
465
    """
479
466
 
480
467
    repository_class = KnitPackRepository
481
 
    _commit_builder_class = PackCommitBuilder
 
468
    _commit_builder_class = PackRootCommitBuilder
482
469
    rich_root_data = True
483
470
    experimental = True
484
471
    supports_tree_reference = True
492
479
        return xml7.serializer_v7
493
480
 
494
481
    def _get_matching_bzrdir(self):
495
 
        return controldir.format_registry.make_controldir(
 
482
        return controldir.format_registry.make_bzrdir(
496
483
            'development5-subtree')
497
484
 
498
485
    def _ignore_setting_bzrdir(self, format):
499
486
        pass
500
487
 
501
 
    _matchingcontroldir = property(
502
 
        _get_matching_bzrdir, _ignore_setting_bzrdir)
 
488
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
503
489
 
504
490
    @classmethod
505
491
    def get_format_string(cls):
506
492
        """See RepositoryFormat.get_format_string()."""
507
 
        return (b"Bazaar development format 2 with subtree support "
508
 
                b"(needs bzr.dev from before 1.8)\n")
 
493
        return ("Bazaar development format 2 with subtree support "
 
494
            "(needs bzr.dev from before 1.8)\n")
509
495
 
510
496
    def get_format_description(self):
511
497
        """See RepositoryFormat.get_format_description()."""
512
498
        return ("Development repository format, currently the same as "
513
 
                "1.6.1-subtree with B+Tree indices.\n")
 
499
            "1.6.1-subtree with B+Tree indices.\n")
514
500
 
515
501
 
516
502
class KnitPackStreamSource(StreamSource):
540
526
        content_text_keys = set()
541
527
        knit = KnitVersionedFiles(None, None)
542
528
        factory = KnitPlainFactory()
543
 
 
544
529
        def find_text_keys_from_content(record):
545
530
            if record.storage_kind not in ('knit-delta-gz', 'knit-ft-gz'):
546
531
                raise ValueError("Unknown content storage kind for"
547
 
                                 " inventory text: %s" % (record.storage_kind,))
 
532
                    " inventory text: %s" % (record.storage_kind,))
548
533
            # It's a knit record, it has a _raw_record field (even if it was
549
534
            # reconstituted from a network stream).
550
535
            raw_data = record._raw_record
558
543
            content_text_keys.update(find_text_keys(
559
544
                [(line, revision_id) for line in line_iterator]))
560
545
        revision_keys = [(r,) for r in revision_ids]
561
 
 
562
546
        def _filtered_inv_stream():
563
547
            source_vf = from_repo.inventories
564
548
            stream = source_vf.get_record_stream(revision_keys,
575
559
        # Note: We know we don't have to handle adding root keys, because both
576
560
        # the source and target are the identical network name.
577
561
        text_stream = self.from_repository.texts.get_record_stream(
578
 
            self._text_keys, self._text_fetch_order, False)
 
562
                        self._text_keys, self._text_fetch_order, False)
579
563
        return ('texts', text_stream)
580
564
 
581
565
    def get_stream(self, search):
593
577
    def __init__(self, pack_collection, packs, suffix, revision_ids=None,
594
578
                 reload_func=None):
595
579
        super(KnitPacker, self).__init__(pack_collection, packs, suffix,
596
 
                                         revision_ids=revision_ids,
597
 
                                         reload_func=reload_func)
 
580
                                          revision_ids=revision_ids,
 
581
                                          reload_func=reload_func)
598
582
 
599
583
    def _pack_map_and_index_list(self, index_attribute):
600
584
        """Convert a list of packs to an index pack map and index list.
626
610
            return all_index.iter_entries(key_filter)
627
611
 
628
612
    def _copy_nodes(self, nodes, index_map, writer, write_index,
629
 
                    output_lines=None):
 
613
        output_lines=None):
630
614
        """Copy knit nodes between packs with no graph references.
631
615
 
632
616
        :param output_lines: Output full texts of copied items.
633
617
        """
634
 
        with ui.ui_factory.nested_progress_bar() as pb:
 
618
        pb = ui.ui_factory.nested_progress_bar()
 
619
        try:
635
620
            return self._do_copy_nodes(nodes, index_map, writer,
636
 
                                       write_index, pb, output_lines=output_lines)
 
621
                write_index, pb, output_lines=output_lines)
 
622
        finally:
 
623
            pb.finished()
637
624
 
638
625
    def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb,
639
 
                       output_lines=None):
 
626
        output_lines=None):
640
627
        # for record verification
641
628
        knit = KnitVersionedFiles(None, None)
642
629
        # plan a readv on each source pack:
653
640
            request_groups[index].append((key, value))
654
641
        record_index = 0
655
642
        pb.update("Copied record", record_index, len(nodes))
656
 
        for index, items in request_groups.items():
 
643
        for index, items in request_groups.iteritems():
657
644
            pack_readv_requests = []
658
645
            for key, value in items:
659
646
                # ---- KnitGraphIndex.get_position
660
 
                bits = value[1:].split(b' ')
 
647
                bits = value[1:].split(' ')
661
648
                offset, length = int(bits[0]), int(bits[1])
662
 
                pack_readv_requests.append((offset, length, (key, value[0:1])))
 
649
                pack_readv_requests.append((offset, length, (key, value[0])))
663
650
            # linear scan up the pack
664
651
            pack_readv_requests.sort()
665
652
            # copy the data
667
654
            transport, path = pack_obj.access_tuple()
668
655
            try:
669
656
                reader = pack.make_readv_reader(transport, path,
670
 
                                                [offset[0:2] for offset in pack_readv_requests])
 
657
                    [offset[0:2] for offset in pack_readv_requests])
671
658
            except errors.NoSuchFile:
672
659
                if self._reload_func is not None:
673
660
                    self._reload_func()
674
661
                raise
675
 
            for (names, read_func), (_1, _2, (key, eol_flag)) in zip(
676
 
                    reader.iter_records(), pack_readv_requests):
 
662
            for (names, read_func), (_1, _2, (key, eol_flag)) in \
 
663
                izip(reader.iter_records(), pack_readv_requests):
677
664
                raw_data = read_func(None)
678
665
                # check the header only
679
666
                if output_lines is not None:
681
668
                else:
682
669
                    df, _ = knit._parse_record_header(key, raw_data)
683
670
                    df.close()
684
 
                pos, size = writer.add_bytes_record([raw_data], len(raw_data), names)
685
 
                write_index.add_node(key, eol_flag + b"%d %d" % (pos, size))
 
671
                pos, size = writer.add_bytes_record(raw_data, names)
 
672
                write_index.add_node(key, eol_flag + "%d %d" % (pos, size))
686
673
                pb.update("Copied record", record_index)
687
674
                record_index += 1
688
675
 
689
676
    def _copy_nodes_graph(self, index_map, writer, write_index,
690
 
                          readv_group_iter, total_items, output_lines=False):
 
677
        readv_group_iter, total_items, output_lines=False):
691
678
        """Copy knit nodes between packs.
692
679
 
693
680
        :param output_lines: Return lines present in the copied data as
694
681
            an iterator of line,version_id.
695
682
        """
696
 
        with ui.ui_factory.nested_progress_bar() as pb:
 
683
        pb = ui.ui_factory.nested_progress_bar()
 
684
        try:
697
685
            for result in self._do_copy_nodes_graph(index_map, writer,
698
 
                                                    write_index, output_lines, pb, readv_group_iter, total_items):
 
686
                write_index, output_lines, pb, readv_group_iter, total_items):
699
687
                yield result
 
688
        except Exception:
 
689
            # Python 2.4 does not permit try:finally: in a generator.
 
690
            pb.finished()
 
691
            raise
 
692
        else:
 
693
            pb.finished()
700
694
 
701
695
    def _do_copy_nodes_graph(self, index_map, writer, write_index,
702
 
                             output_lines, pb, readv_group_iter, total_items):
 
696
        output_lines, pb, readv_group_iter, total_items):
703
697
        # for record verification
704
698
        knit = KnitVersionedFiles(None, None)
705
699
        # for line extraction when requested (inventories only)
717
711
                if self._reload_func is not None:
718
712
                    self._reload_func()
719
713
                raise
720
 
            for (names, read_func), (key, eol_flag, references) in zip(
721
 
                    reader.iter_records(), node_vector):
 
714
            for (names, read_func), (key, eol_flag, references) in \
 
715
                izip(reader.iter_records(), node_vector):
722
716
                raw_data = read_func(None)
723
717
                if output_lines:
724
718
                    # read the entire thing
733
727
                    # check the header only
734
728
                    df, _ = knit._parse_record_header(key, raw_data)
735
729
                    df.close()
736
 
                pos, size = writer.add_bytes_record([raw_data], len(raw_data), names)
737
 
                write_index.add_node(key, eol_flag + b"%d %d" %
738
 
                                     (pos, size), references)
 
730
                pos, size = writer.add_bytes_record(raw_data, names)
 
731
                write_index.add_node(key, eol_flag + "%d %d" % (pos, size), references)
739
732
                pb.update("Copied record", record_index)
740
733
                record_index += 1
741
734
 
745
738
        fileid_revisions = repo._find_file_ids_from_xml_inventory_lines(
746
739
            inv_lines, self.revision_keys)
747
740
        text_filter = []
748
 
        for fileid, file_revids in fileid_revisions.items():
749
 
            text_filter.extend([(fileid, file_revid)
750
 
                                for file_revid in file_revids])
 
741
        for fileid, file_revids in fileid_revisions.iteritems():
 
742
            text_filter.extend([(fileid, file_revid) for file_revid in file_revids])
751
743
        self._text_filter = text_filter
752
744
 
753
745
    def _copy_inventory_texts(self):
754
746
        # select inventory keys
755
 
        inv_keys = self._revision_keys  # currently the same keyspace, and note that
 
747
        inv_keys = self._revision_keys # currently the same keyspace, and note that
756
748
        # querying for keys here could introduce a bug where an inventory item
757
749
        # is missed, so do not change it to query separately without cross
758
750
        # checking like the text key check below.
767
759
        # Only grab the output lines if we will be processing them
768
760
        output_lines = bool(self.revision_ids)
769
761
        inv_lines = self._copy_nodes_graph(inventory_index_map,
770
 
                                           self.new_pack._writer, self.new_pack.inventory_index,
771
 
                                           readv_group_iter, total_items, output_lines=output_lines)
 
762
            self.new_pack._writer, self.new_pack.inventory_index,
 
763
            readv_group_iter, total_items, output_lines=output_lines)
772
764
        if self.revision_ids:
773
765
            self._process_inventory_lines(inv_lines)
774
766
        else:
777
769
            self._text_filter = None
778
770
        if 'pack' in debug.debug_flags:
779
771
            trace.mutter('%s: create_pack: inventories copied: %s%s %d items t+%6.3fs',
780
 
                         time.ctime(), self._pack_collection._upload_transport.base,
781
 
                         self.new_pack.random_name,
782
 
                         self.new_pack.inventory_index.key_count(),
783
 
                         time.time() - self.new_pack.start_time)
 
772
                time.ctime(), self._pack_collection._upload_transport.base,
 
773
                self.new_pack.random_name,
 
774
                self.new_pack.inventory_index.key_count(),
 
775
                time.time() - self.new_pack.start_time)
784
776
 
785
777
    def _update_pack_order(self, entries, index_to_pack_map):
786
778
        """Determine how we want our packs to be ordered.
813
805
            old_names = [p.access_tuple()[1] for p in self.packs]
814
806
            new_names = [p.access_tuple()[1] for p in packs]
815
807
            trace.mutter('Reordering packs\nfrom: %s\n  to: %s',
816
 
                         old_names, new_names)
 
808
                   old_names, new_names)
817
809
        self.packs = packs
818
810
 
819
811
    def _copy_revision_texts(self):
820
812
        # select revisions
821
813
        if self.revision_ids:
822
 
            revision_keys = [(revision_id,)
823
 
                             for revision_id in self.revision_ids]
 
814
            revision_keys = [(revision_id,) for revision_id in self.revision_ids]
824
815
        else:
825
816
            revision_keys = None
826
817
        # select revision keys
831
822
        self._update_pack_order(revision_nodes, revision_index_map)
832
823
        # copy revision keys and adjust values
833
824
        self.pb.update("Copying revision texts", 1)
834
 
        total_items, readv_group_iter = self._revision_node_readv(
835
 
            revision_nodes)
 
825
        total_items, readv_group_iter = self._revision_node_readv(revision_nodes)
836
826
        list(self._copy_nodes_graph(revision_index_map, self.new_pack._writer,
837
 
                                    self.new_pack.revision_index, readv_group_iter, total_items))
 
827
            self.new_pack.revision_index, readv_group_iter, total_items))
838
828
        if 'pack' in debug.debug_flags:
839
829
            trace.mutter('%s: create_pack: revisions copied: %s%s %d items t+%6.3fs',
840
 
                         time.ctime(), self._pack_collection._upload_transport.base,
841
 
                         self.new_pack.random_name,
842
 
                         self.new_pack.revision_index.key_count(),
843
 
                         time.time() - self.new_pack.start_time)
 
830
                time.ctime(), self._pack_collection._upload_transport.base,
 
831
                self.new_pack.random_name,
 
832
                self.new_pack.revision_index.key_count(),
 
833
                time.time() - self.new_pack.start_time)
844
834
        self._revision_keys = revision_keys
845
835
 
846
836
    def _get_text_nodes(self):
847
837
        text_index_map, text_indices = self._pack_map_and_index_list(
848
838
            'text_index')
849
839
        return text_index_map, self._index_contents(text_indices,
850
 
                                                    self._text_filter)
 
840
            self._text_filter)
851
841
 
852
842
    def _copy_text_texts(self):
853
843
        # select text keys
865
855
            if missing_text_keys:
866
856
                # TODO: raise a specific error that can handle many missing
867
857
                # keys.
868
 
                trace.mutter("missing keys during fetch: %r",
869
 
                             missing_text_keys)
 
858
                trace.mutter("missing keys during fetch: %r", missing_text_keys)
870
859
                a_missing_key = missing_text_keys.pop()
871
860
                raise errors.RevisionNotPresent(a_missing_key[1],
872
 
                                                a_missing_key[0])
 
861
                    a_missing_key[0])
873
862
        # copy text keys and adjust values
874
863
        self.pb.update("Copying content texts", 3)
875
 
        total_items, readv_group_iter = self._least_readv_node_readv(
876
 
            text_nodes)
 
864
        total_items, readv_group_iter = self._least_readv_node_readv(text_nodes)
877
865
        list(self._copy_nodes_graph(text_index_map, self.new_pack._writer,
878
 
                                    self.new_pack.text_index, readv_group_iter, total_items))
 
866
            self.new_pack.text_index, readv_group_iter, total_items))
879
867
        self._log_copied_texts()
880
868
 
881
869
    def _create_pack_from_packs(self):
884
872
        new_pack = self.new_pack
885
873
        # buffer data - we won't be reading-back during the pack creation and
886
874
        # this makes a significant difference on sftp pushes.
887
 
        new_pack.set_write_cache_size(1024 * 1024)
 
875
        new_pack.set_write_cache_size(1024*1024)
888
876
        if 'pack' in debug.debug_flags:
889
877
            plain_pack_list = ['%s%s' % (a_pack.pack_transport.base, a_pack.name)
890
 
                               for a_pack in self.packs]
 
878
                for a_pack in self.packs]
891
879
            if self.revision_ids is not None:
892
880
                rev_count = len(self.revision_ids)
893
881
            else:
894
882
                rev_count = 'all'
895
883
            trace.mutter('%s: create_pack: creating pack from source packs: '
896
 
                         '%s%s %s revisions wanted %s t=0',
897
 
                         time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
898
 
                         plain_pack_list, rev_count)
 
884
                '%s%s %s revisions wanted %s t=0',
 
885
                time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
 
886
                plain_pack_list, rev_count)
899
887
        self._copy_revision_texts()
900
888
        self._copy_inventory_texts()
901
889
        self._copy_text_texts()
902
890
        # select signature keys
903
 
        signature_filter = self._revision_keys  # same keyspace
 
891
        signature_filter = self._revision_keys # same keyspace
904
892
        signature_index_map, signature_indices = self._pack_map_and_index_list(
905
893
            'signature_index')
906
894
        signature_nodes = self._index_contents(signature_indices,
907
 
                                               signature_filter)
 
895
            signature_filter)
908
896
        # copy signature keys and adjust values
909
897
        self.pb.update("Copying signature texts", 4)
910
898
        self._copy_nodes(signature_nodes, signature_index_map, new_pack._writer,
911
 
                         new_pack.signature_index)
 
899
            new_pack.signature_index)
912
900
        if 'pack' in debug.debug_flags:
913
901
            trace.mutter('%s: create_pack: revision signatures copied: %s%s %d items t+%6.3fs',
914
 
                         time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
915
 
                         new_pack.signature_index.key_count(),
916
 
                         time.time() - new_pack.start_time)
 
902
                time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
 
903
                new_pack.signature_index.key_count(),
 
904
                time.time() - new_pack.start_time)
917
905
        new_pack._check_references()
918
906
        if not self._use_pack(new_pack):
919
907
            new_pack.abort()
944
932
                request_groups[index] = []
945
933
            request_groups[index].append((key, value, references))
946
934
        result = []
947
 
        for index, items in request_groups.items():
 
935
        for index, items in request_groups.iteritems():
948
936
            pack_readv_requests = []
949
937
            for key, value, references in items:
950
938
                # ---- KnitGraphIndex.get_position
951
 
                bits = value[1:].split(b' ')
 
939
                bits = value[1:].split(' ')
952
940
                offset, length = int(bits[0]), int(bits[1])
953
941
                pack_readv_requests.append(
954
 
                    ((offset, length), (key, value[0:1], references)))
 
942
                    ((offset, length), (key, value[0], references)))
955
943
            # linear scan up the pack to maximum range combining.
956
944
            pack_readv_requests.sort()
957
945
            # split out the readv and the node data.
973
961
class KnitReconcilePacker(KnitPacker):
974
962
    """A packer which regenerates indices etc as it copies.
975
963
 
976
 
    This is used by ``brz reconcile`` to cause parent text pointers to be
 
964
    This is used by ``bzr reconcile`` to cause parent text pointers to be
977
965
    regenerated.
978
966
    """
979
967
 
1001
989
        # 1) generate the ideal index
1002
990
        repo = self._pack_collection.repo
1003
991
        ancestors = dict([(key[0], tuple(ref[0] for ref in refs[0])) for
1004
 
                          _1, key, _2, refs in
1005
 
                          self.new_pack.revision_index.iter_all_entries()])
 
992
            _1, key, _2, refs in
 
993
            self.new_pack.revision_index.iter_all_entries()])
1006
994
        ideal_index = repo._generate_text_key_index(self._text_refs, ancestors)
1007
995
        # 2) generate a text_nodes list that contains all the deltas that can
1008
996
        #    be used as-is, with corrected parents.
1033
1021
                    # long as we change the refs to be inserted.
1034
1022
                    self._data_changed = True
1035
1023
                    ok_nodes.append((node[0], node[1], node[2],
1036
 
                                     (ideal_parents, node[3][1])))
 
1024
                        (ideal_parents, node[3][1])))
1037
1025
                    self._data_changed = True
1038
1026
                else:
1039
1027
                    # Reinsert this text completely
1045
1033
        # 3) bulk copy the ok data
1046
1034
        total_items, readv_group_iter = self._least_readv_node_readv(ok_nodes)
1047
1035
        list(self._copy_nodes_graph(text_index_map, self.new_pack._writer,
1048
 
                                    self.new_pack.text_index, readv_group_iter, total_items))
 
1036
            self.new_pack.text_index, readv_group_iter, total_items))
1049
1037
        # 4) adhoc copy all the other texts.
1050
1038
        # We have to topologically insert all texts otherwise we can fail to
1051
1039
        # reconcile when parts of a single delta chain are preserved intact,
1059
1047
        # space (we only topo sort the revisions, which is smaller).
1060
1048
        topo_order = tsort.topo_sort(ancestors)
1061
1049
        rev_order = dict(zip(topo_order, range(len(topo_order))))
1062
 
        bad_texts.sort(key=lambda key: rev_order.get(key[0][1], 0))
 
1050
        bad_texts.sort(key=lambda key:rev_order.get(key[0][1], 0))
1063
1051
        transaction = repo.get_transaction()
1064
1052
        file_id_index = GraphIndexPrefixAdapter(
1065
1053
            self.new_pack.text_index,
1066
1054
            ('blank', ), 1,
1067
1055
            add_nodes_callback=self.new_pack.text_index.add_nodes)
1068
1056
        data_access = _DirectPackAccess(
1069
 
            {self.new_pack.text_index: self.new_pack.access_tuple()})
 
1057
                {self.new_pack.text_index:self.new_pack.access_tuple()})
1070
1058
        data_access.set_writer(self.new_pack._writer, self.new_pack.text_index,
1071
 
                               self.new_pack.access_tuple())
 
1059
            self.new_pack.access_tuple())
1072
1060
        output_texts = KnitVersionedFiles(
1073
1061
            _KnitGraphIndex(self.new_pack.text_index,
1074
 
                            add_callback=self.new_pack.text_index.add_nodes,
1075
 
                            deltas=True, parents=True, is_locked=repo.is_locked),
 
1062
                add_callback=self.new_pack.text_index.add_nodes,
 
1063
                deltas=True, parents=True, is_locked=repo.is_locked),
1076
1064
            data_access=data_access, max_delta_chain=200)
1077
1065
        for key, parent_keys in bad_texts:
1078
1066
            # We refer to the new pack to delta data being output.
1084
1072
                if parent_key[0] != key[0]:
1085
1073
                    # Graph parents must match the fileid
1086
1074
                    raise errors.BzrError('Mismatched key parent %r:%r' %
1087
 
                                          (key, parent_keys))
 
1075
                        (key, parent_keys))
1088
1076
                parents.append(parent_key[1])
1089
 
            text_lines = next(repo.texts.get_record_stream(
1090
 
                [key], 'unordered', True)).get_bytes_as('lines')
 
1077
            text_lines = osutils.split_lines(repo.texts.get_record_stream(
 
1078
                [key], 'unordered', True).next().get_bytes_as('fulltext'))
1091
1079
            output_texts.add_lines(key, parent_keys, text_lines,
1092
 
                                   random_id=True, check_content=False)
 
1080
                random_id=True, check_content=False)
1093
1081
        # 5) check that nothing inserted has a reference outside the keyspace.
1094
1082
        missing_text_keys = self.new_pack.text_index._external_references()
1095
1083
        if missing_text_keys:
1096
1084
            raise errors.BzrCheckError('Reference to missing compression parents %r'
1097
 
                                       % (missing_text_keys,))
 
1085
                % (missing_text_keys,))
1098
1086
        self._log_copied_texts()
1099
1087
 
1100
1088
    def _use_pack(self, new_pack):
1138
1126
        for key in reversed(order):
1139
1127
            index, value, references = by_key[key]
1140
1128
            # ---- KnitGraphIndex.get_position
1141
 
            bits = value[1:].split(b' ')
 
1129
            bits = value[1:].split(' ')
1142
1130
            offset, length = int(bits[0]), int(bits[1])
1143
1131
            requests.append(
1144
 
                (index, [(offset, length)], [(key, value[0:1], references)]))
 
1132
                (index, [(offset, length)], [(key, value[0], references)]))
1145
1133
        # TODO: combine requests in the same index that are in ascending order.
1146
1134
        return total, requests
1147
1135
 
1163
1151
    resumed_pack_factory = ResumedPack
1164
1152
    normal_packer_class = KnitPacker
1165
1153
    optimising_packer_class = OptimisingKnitPacker
 
1154
 
 
1155
 
 
1156