To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to bzrlib/repofmt/pack_repo.py
- browse files at revision 3794.5.24
- compare with another revision
- compare with revision 0.200.231
- download diff
- download tarball
- view history from revision 3794.5.24
- Committer: Mark Hammond
- Date: 2008-12-21 07:42:20 UTC
- mfrom: (3915 +trunk)
- mto: (3932.3.1 cicp-1.11)
- mto: This revision was merged to the branch mainline in revision 3937.
- Revision ID: mhammond@skippinet.com.au-20081221074220-7dr5oydglxyyvic3
Merge trunk.
- files added:
- bzrlib/_chunks_to_lines_py.py
- files modified:
- .bzrignore
added
removed
bzrlib/repofmt/pack_repo.py
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
16
17
import sys
18
17
19
from bzrlib.lazy_import import lazy_import
18
20
lazy_import(globals(), """
19
21
from itertools import izip
172
174
"""The text index is the name + .tix."""
173
175
return self.index_name('text', name)
174
176
175
def _external_compression_parents_of_texts(self):
176
keys = set()
177
refs = set()
178
for node in self.text_index.iter_all_entries():
179
keys.add(node[1])
180
refs.update(node[3][1])
181
return refs - keys
182
183
177
184
178
class ExistingPack(Pack):
185
179
"""An in memory proxy for an existing .pack and its disk indices."""
222
216
'signature': ('.six', 3),
223
217
}
224
218
225
def __init__(self, upload_transport, index_transport, pack_transport,
226
upload_suffix='', file_mode=None, index_builder_class=None,
227
index_class=None):
219
def __init__(self, pack_collection, upload_suffix='', file_mode=None):
228
220
"""Create a NewPack instance.
229
221
230
:param upload_transport: A writable transport for the pack to be
231
incrementally uploaded to.
232
:param index_transport: A writable transport for the pack's indices to
233
be written to when the pack is finished.
234
:param pack_transport: A writable transport for the pack to be renamed
235
to when the upload is complete. This *must* be the same as
236
upload_transport.clone('../packs').
222
:param pack_collection: A PackCollection into which this is being inserted.
237
223
:param upload_suffix: An optional suffix to be given to any temporary
238
224
files created during the pack creation. e.g '.autopack'
239
:param file_mode: An optional file mode to create the new files with.
240
:param index_builder_class: Required keyword parameter - the class of
241
index builder to use.
242
:param index_class: Required keyword parameter - the class of index
243
object to use.
225
:param file_mode: Unix permissions for newly created file.
244
226
"""
245
227
# The relative locations of the packs are constrained, but all are
246
228
# passed in because the caller has them, so as to avoid object churn.
229
index_builder_class = pack_collection._index_builder_class
247
230
Pack.__init__(self,
248
231
# Revisions: parents list, no text compression.
249
232
index_builder_class(reference_lists=1),
259
242
# listing.
260
243
index_builder_class(reference_lists=0),
261
244
)
245
self._pack_collection = pack_collection
262
246
# When we make readonly indices, we need this.
263
self.index_class = index_class
247
self.index_class = pack_collection._index_class
264
248
# where should the new pack be opened
265
self.upload_transport = upload_transport
249
self.upload_transport = pack_collection._upload_transport
266
250
# where are indices written out to
267
self.index_transport = index_transport
251
self.index_transport = pack_collection._index_transport
268
252
# where is the pack renamed to when it is finished?
269
self.pack_transport = pack_transport
253
self.pack_transport = pack_collection._pack_transport
270
254
# What file mode to upload the pack and indices with.
271
255
self._file_mode = file_mode
272
256
# tracks the content written to the .pack file.
334
318
else:
335
319
raise AssertionError(self._state)
336
320
321
def _check_references(self):
322
"""Make sure our external references are present.
323
324
Packs are allowed to have deltas whose base is not in the pack, but it
325
must be present somewhere in this collection. It is not allowed to
326
have deltas based on a fallback repository.
327
(See <https://bugs.launchpad.net/bzr/+bug/288751>)
328
"""
329
missing_items = {}
330
for (index_name, external_refs, index) in [
331
('texts',
332
self.text_index._external_references(),
333
self._pack_collection.text_index.combined_index),
334
('inventories',
335
self.inventory_index._external_references(),
336
self._pack_collection.inventory_index.combined_index),
337
]:
338
missing = external_refs.difference(
339
k for (idx, k, v, r) in
340
index.iter_entries(external_refs))
341
if missing:
342
missing_items[index_name] = sorted(list(missing))
343
if missing_items:
344
from pprint import pformat
345
raise errors.BzrCheckError(
346
"Newly created pack file %r has delta references to "
347
"items not in its repository:\n%s"
348
% (self, pformat(missing_items)))
349
337
350
def data_inserted(self):
338
351
"""True if data has been added to this pack."""
339
352
return bool(self.get_revision_count() or
356
369
if self._buffer[1]:
357
370
self._write_data('', flush=True)
358
371
self.name = self._hash.hexdigest()
372
self._check_references()
359
373
# write indices
360
374
# XXX: It'd be better to write them all to temporary names, then
361
375
# rename them all into place, so that the window when only some are
462
476
self._reload_func = reload_func
463
477
self.index_to_pack = {}
464
478
self.combined_index = CombinedGraphIndex([], reload_func=reload_func)
465
self.data_access = _DirectPackAccess(self.index_to_pack)
479
self.data_access = _DirectPackAccess(self.index_to_pack,
480
reload_func=reload_func)
466
481
self.add_callback = None
467
482
468
483
def replace_indices(self, index_to_pack, indices):
542
557
class Packer(object):
543
558
"""Create a pack from packs."""
544
559
545
def __init__(self, pack_collection, packs, suffix, revision_ids=None):
560
def __init__(self, pack_collection, packs, suffix, revision_ids=None,
561
reload_func=None):
546
562
"""Create a Packer.
547
563
548
564
:param pack_collection: A RepositoryPackCollection object where the
550
566
:param packs: The packs to combine.
551
567
:param suffix: The suffix to use on the temporary files for the pack.
552
568
:param revision_ids: Revision ids to limit the pack to.
569
:param reload_func: A function to call if a pack file/index goes
570
missing. The side effect of calling this function should be to
571
update self.packs. See also AggregateIndex
553
572
"""
554
573
self.packs = packs
555
574
self.suffix = suffix
557
576
# The pack object we are creating.
558
577
self.new_pack = None
559
578
self._pack_collection = pack_collection
579
self._reload_func = reload_func
560
580
# The index layer keys for the revisions being copied. None for 'all
561
581
# objects'.
562
582
self._revision_keys = None
568
588
def _extra_init(self):
569
589
"""A template hook to allow extending the constructor trivially."""
570
590
591
def _pack_map_and_index_list(self, index_attribute):
592
"""Convert a list of packs to an index pack map and index list.
593
594
:param index_attribute: The attribute that the desired index is found
595
on.
596
:return: A tuple (map, list) where map contains the dict from
597
index:pack_tuple, and list contains the indices in the preferred
598
access order.
599
"""
600
indices = []
601
pack_map = {}
602
for pack_obj in self.packs:
603
index = getattr(pack_obj, index_attribute)
604
indices.append(index)
605
pack_map[index] = pack_obj
606
return pack_map, indices
607
608
def _index_contents(self, indices, key_filter=None):
609
"""Get an iterable of the index contents from a pack_map.
610
611
:param indices: The list of indices to query
612
:param key_filter: An optional filter to limit the keys returned.
613
"""
614
all_index = CombinedGraphIndex(indices)
615
if key_filter is None:
616
return all_index.iter_all_entries()
617
else:
618
return all_index.iter_entries(key_filter)
619
571
620
def pack(self, pb=None):
572
621
"""Create a new pack by reading data from other packs.
573
622
587
636
# XXX: - duplicate code warning with start_write_group; fix before
588
637
# considering 'done'.
589
638
if self._pack_collection._new_pack is not None:
590
raise errors.BzrError('call to create_pack_from_packs while '
591
'another pack is being written.')
639
raise errors.BzrError('call to %s.pack() while another pack is'
640
' being written.'
641
% (self.__class__.__name__,))
592
642
if self.revision_ids is not None:
593
643
if len(self.revision_ids) == 0:
594
644
# silly fetch request.
609
659
610
660
def open_pack(self):
611
661
"""Open a pack for the pack we are creating."""
612
return NewPack(self._pack_collection._upload_transport,
613
self._pack_collection._index_transport,
614
self._pack_collection._pack_transport, upload_suffix=self.suffix,
615
file_mode=self._pack_collection.repo.bzrdir._get_file_mode(),
616
index_builder_class=self._pack_collection._index_builder_class,
617
index_class=self._pack_collection._index_class)
662
return NewPack(self._pack_collection, upload_suffix=self.suffix,
663
file_mode=self._pack_collection.repo.bzrdir._get_file_mode())
664
665
def _update_pack_order(self, entries, index_to_pack_map):
666
"""Determine how we want our packs to be ordered.
667
668
This changes the sort order of the self.packs list so that packs unused
669
by 'entries' will be at the end of the list, so that future requests
670
can avoid probing them. Used packs will be at the front of the
671
self.packs list, in the order of their first use in 'entries'.
672
673
:param entries: A list of (index, ...) tuples
674
:param index_to_pack_map: A mapping from index objects to pack objects.
675
"""
676
packs = []
677
seen_indexes = set()
678
for entry in entries:
679
index = entry[0]
680
if index not in seen_indexes:
681
packs.append(index_to_pack_map[index])
682
seen_indexes.add(index)
683
if len(packs) == len(self.packs):
684
if 'pack' in debug.debug_flags:
685
mutter('Not changing pack list, all packs used.')
686
return
687
seen_packs = set(packs)
688
for pack in self.packs:
689
if pack not in seen_packs:
690
packs.append(pack)
691
seen_packs.add(pack)
692
if 'pack' in debug.debug_flags:
693
old_names = [p.access_tuple()[1] for p in self.packs]
694
new_names = [p.access_tuple()[1] for p in packs]
695
mutter('Reordering packs\nfrom: %s\n to: %s',
696
old_names, new_names)
697
self.packs = packs
618
698
619
699
def _copy_revision_texts(self):
620
700
"""Copy revision data to the new pack."""
624
704
else:
625
705
revision_keys = None
626
706
# select revision keys
627
revision_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list(
628
self.packs, 'revision_index')[0]
629
revision_nodes = self._pack_collection._index_contents(revision_index_map, revision_keys)
707
revision_index_map, revision_indices = self._pack_map_and_index_list(
708
'revision_index')
709
revision_nodes = self._index_contents(revision_indices, revision_keys)
710
revision_nodes = list(revision_nodes)
711
self._update_pack_order(revision_nodes, revision_index_map)
630
712
# copy revision keys and adjust values
631
713
self.pb.update("Copying revision texts", 1)
632
714
total_items, readv_group_iter = self._revision_node_readv(revision_nodes)
652
734
# querying for keys here could introduce a bug where an inventory item
653
735
# is missed, so do not change it to query separately without cross
654
736
# checking like the text key check below.
655
inventory_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list(
656
self.packs, 'inventory_index')[0]
657
inv_nodes = self._pack_collection._index_contents(inventory_index_map, inv_keys)
737
inventory_index_map, inventory_indices = self._pack_map_and_index_list(
738
'inventory_index')
739
inv_nodes = self._index_contents(inventory_indices, inv_keys)
658
740
# copy inventory keys and adjust values
659
741
# XXX: Should be a helper function to allow different inv representation
660
742
# at this point.
704
786
self.new_pack.text_index, readv_group_iter, total_items))
705
787
self._log_copied_texts()
706
788
707
def _check_references(self):
708
"""Make sure our external refereneces are present."""
709
external_refs = self.new_pack._external_compression_parents_of_texts()
710
if external_refs:
711
index = self._pack_collection.text_index.combined_index
712
found_items = list(index.iter_entries(external_refs))
713
if len(found_items) != len(external_refs):
714
found_keys = set(k for idx, k, refs, value in found_items)
715
missing_items = external_refs - found_keys
716
missing_file_id, missing_revision_id = missing_items.pop()
717
raise errors.RevisionNotPresent(missing_revision_id,
718
missing_file_id)
719
720
789
def _create_pack_from_packs(self):
721
790
self.pb.update("Opening pack", 0, 5)
722
791
self.new_pack = self.open_pack()
740
809
self._copy_text_texts()
741
810
# select signature keys
742
811
signature_filter = self._revision_keys # same keyspace
743
signature_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list(
744
self.packs, 'signature_index')[0]
745
signature_nodes = self._pack_collection._index_contents(signature_index_map,
812
signature_index_map, signature_indices = self._pack_map_and_index_list(
813
'signature_index')
814
signature_nodes = self._index_contents(signature_indices,
746
815
signature_filter)
747
816
# copy signature keys and adjust values
748
817
self.pb.update("Copying signature texts", 4)
753
822
time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
754
823
new_pack.signature_index.key_count(),
755
824
time.time() - new_pack.start_time)
756
self._check_references()
825
new_pack._check_references()
757
826
if not self._use_pack(new_pack):
758
827
new_pack.abort()
759
828
return None
798
867
# linear scan up the pack
799
868
pack_readv_requests.sort()
800
869
# copy the data
801
transport, path = index_map[index]
802
reader = pack.make_readv_reader(transport, path,
803
[offset[0:2] for offset in pack_readv_requests])
870
pack_obj = index_map[index]
871
transport, path = pack_obj.access_tuple()
872
try:
873
reader = pack.make_readv_reader(transport, path,
874
[offset[0:2] for offset in pack_readv_requests])
875
except errors.NoSuchFile:
876
if self._reload_func is not None:
877
self._reload_func()
878
raise
804
879
for (names, read_func), (_1, _2, (key, eol_flag)) in \
805
880
izip(reader.iter_records(), pack_readv_requests):
806
881
raw_data = read_func(None)
842
917
pb.update("Copied record", record_index, total_items)
843
918
for index, readv_vector, node_vector in readv_group_iter:
844
919
# copy the data
845
transport, path = index_map[index]
846
reader = pack.make_readv_reader(transport, path, readv_vector)
920
pack_obj = index_map[index]
921
transport, path = pack_obj.access_tuple()
922
try:
923
reader = pack.make_readv_reader(transport, path, readv_vector)
924
except errors.NoSuchFile:
925
if self._reload_func is not None:
926
self._reload_func()
927
raise
847
928
for (names, read_func), (key, eol_flag, references) in \
848
929
izip(reader.iter_records(), node_vector):
849
930
raw_data = read_func(None)
866
947
record_index += 1
867
948
868
949
def _get_text_nodes(self):
869
text_index_map = self._pack_collection._packs_list_to_pack_map_and_index_list(
870
self.packs, 'text_index')[0]
871
return text_index_map, self._pack_collection._index_contents(text_index_map,
950
text_index_map, text_indices = self._pack_map_and_index_list(
951
'text_index')
952
return text_index_map, self._index_contents(text_indices,
872
953
self._text_filter)
873
954
874
955
def _least_readv_node_readv(self, nodes):
1109
1190
output_texts.add_lines(key, parent_keys, text_lines,
1110
1191
random_id=True, check_content=False)
1111
1192
# 5) check that nothing inserted has a reference outside the keyspace.
1112
missing_text_keys = self.new_pack._external_compression_parents_of_texts()
1193
missing_text_keys = self.new_pack.text_index._external_references()
1113
1194
if missing_text_keys:
1114
raise errors.BzrError('Reference to missing compression parents %r'
1195
raise errors.BzrCheckError('Reference to missing compression parents %r'
1115
1196
% (missing_text_keys,))
1116
1197
self._log_copied_texts()
1117
1198
1212
1293
1213
1294
:return: True if packing took place.
1214
1295
"""
1296
while True:
1297
try:
1298
return self._do_autopack()
1299
except errors.RetryAutopack, e:
1300
# If we get a RetryAutopack exception, we should abort the
1301
# current action, and retry.
1302
pass
1303
1304
def _do_autopack(self):
1215
1305
# XXX: Should not be needed when the management of indices is sane.
1216
1306
total_revisions = self.revision_index.combined_index.key_count()
1217
1307
total_packs = len(self._names)
1219
1309
return False
1220
1310
# XXX: the following may want to be a class, to pack with a given
1221
1311
# policy.
1222
mutter('Auto-packing repository %s, which has %d pack files, '
1223
'containing %d revisions into %d packs.', self, total_packs,
1224
total_revisions, self._max_pack_count(total_revisions))
1225
1312
# determine which packs need changing
1226
1313
pack_distribution = self.pack_distribution(total_revisions)
1227
1314
existing_packs = []
1242
1329
existing_packs.append((revision_count, pack))
1243
1330
pack_operations = self.plan_autopack_combinations(
1244
1331
existing_packs, pack_distribution)
1245
self._execute_pack_operations(pack_operations)
1332
num_new_packs = len(pack_operations)
1333
num_old_packs = sum([len(po[1]) for po in pack_operations])
1334
num_revs_affected = sum([po[0] for po in pack_operations])
1335
mutter('Auto-packing repository %s, which has %d pack files, '
1336
'containing %d revisions. Packing %d files into %d affecting %d'
1337
' revisions', self, total_packs, total_revisions, num_old_packs,
1338
num_new_packs, num_revs_affected)
1339
self._execute_pack_operations(pack_operations,
1340
reload_func=self._restart_autopack)
1246
1341
return True
1247
1342
1248
def _execute_pack_operations(self, pack_operations, _packer_class=Packer):
1343
def _execute_pack_operations(self, pack_operations, _packer_class=Packer,
1344
reload_func=None):
1249
1345
"""Execute a series of pack operations.
1250
1346
1251
1347
:param pack_operations: A list of [revision_count, packs_to_combine].
1256
1352
# we may have no-ops from the setup logic
1257
1353
if len(packs) == 0:
1258
1354
continue
1259
_packer_class(self, packs, '.autopack').pack()
1355
packer = _packer_class(self, packs, '.autopack',
1356
reload_func=reload_func)
1357
try:
1358
packer.pack()
1359
except errors.RetryWithNewPacks:
1360
# An exception is propagating out of this context, make sure
1361
# this packer has cleaned up. Packer() doesn't set its new_pack
1362
# state into the RepositoryPackCollection object, so we only
1363
# have access to it directly here.
1364
if packer.new_pack is not None:
1365
packer.new_pack.abort()
1366
raise
1260
1367
for pack in packs:
1261
1368
self._remove_pack_from_memory(pack)
1262
1369
# record the newly available packs and stop advertising the old
1515
1622
self._packs_by_name = {}
1516
1623
self._packs_at_load = None
1517
1624
1518
def _make_index_map(self, index_suffix):
1519
"""Return information on existing indices.
1520
1521
:param suffix: Index suffix added to pack name.
1522
1523
:returns: (pack_map, indices) where indices is a list of GraphIndex
1524
objects, and pack_map is a mapping from those objects to the
1525
pack tuple they describe.
1526
"""
1527
# TODO: stop using this; it creates new indices unnecessarily.
1528
self.ensure_loaded()
1529
suffix_map = {'.rix': 'revision_index',
1530
'.six': 'signature_index',
1531
'.iix': 'inventory_index',
1532
'.tix': 'text_index',
1533
}
1534
return self._packs_list_to_pack_map_and_index_list(self.all_packs(),
1535
suffix_map[index_suffix])
1536
1537
def _packs_list_to_pack_map_and_index_list(self, packs, index_attribute):
1538
"""Convert a list of packs to an index pack map and index list.
1539
1540
:param packs: The packs list to process.
1541
:param index_attribute: The attribute that the desired index is found
1542
on.
1543
:return: A tuple (map, list) where map contains the dict from
1544
index:pack_tuple, and lsit contains the indices in the same order
1545
as the packs list.
1546
"""
1547
indices = []
1548
pack_map = {}
1549
for pack in packs:
1550
index = getattr(pack, index_attribute)
1551
indices.append(index)
1552
pack_map[index] = (pack.pack_transport, pack.file_name())
1553
return pack_map, indices
1554
1555
def _index_contents(self, pack_map, key_filter=None):
1556
"""Get an iterable of the index contents from a pack_map.
1557
1558
:param pack_map: A map from indices to pack details.
1559
:param key_filter: An optional filter to limit the
1560
keys returned.
1561
"""
1562
indices = [index for index in pack_map.iterkeys()]
1563
all_index = CombinedGraphIndex(indices)
1564
if key_filter is None:
1565
return all_index.iter_all_entries()
1566
else:
1567
return all_index.iter_entries(key_filter)
1568
1569
1625
def _unlock_names(self):
1570
1626
"""Release the mutex around the pack-names index."""
1571
1627
self.repo.control_files.unlock()
1694
1750
return True
1695
1751
return False
1696
1752
1753
def _restart_autopack(self):
1754
"""Reload the pack names list, and restart the autopack code."""
1755
if not self.reload_pack_names():
1756
# Re-raise the original exception, because something went missing
1757
# and a restart didn't find it
1758
raise
1759
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
1760
1697
1761
def _clear_obsolete_packs(self):
1698
1762
"""Delete everything from the obsolete-packs directory.
1699
1763
"""
1708
1772
# Do not permit preparation for writing if we're not in a 'write lock'.
1709
1773
if not self.repo.is_write_locked():
1710
1774
raise errors.NotWriteLocked(self)
1711
self._new_pack = NewPack(self._upload_transport, self._index_transport,
1712
self._pack_transport, upload_suffix='.pack',
1713
file_mode=self.repo.bzrdir._get_file_mode(),
1714
index_builder_class=self._index_builder_class,
1715
index_class=self._index_class)
1775
self._new_pack = NewPack(self, upload_suffix='.pack',
1776
file_mode=self.repo.bzrdir._get_file_mode())
1716
1777
# allow writing: queue writes to a new index
1717
1778
self.revision_index.add_writable_index(self._new_pack.revision_index,
1718
1779
self._new_pack)
1735
1796
try:
1736
1797
self._new_pack.abort()
1737
1798
finally:
1799
# XXX: If we aborted while in the middle of finishing the write
1800
# group, _remove_pack_indices can fail because the indexes are
1801
# already gone. If they're not there we shouldn't fail in this
1802
# case. -- mbp 20081113
1738
1803
self._remove_pack_indices(self._new_pack)
1739
1804
self._new_pack = None
1740
1805
self.repo._text_knit = None
2296
2361
return xml7.serializer_v7
2297
2362
2298
2363
def _get_matching_bzrdir(self):
2299
return bzrdir.format_registry.make_bzrdir(
2364
matching = bzrdir.format_registry.make_bzrdir(
2300
2365
'1.6.1-rich-root')
2366
matching.repository_format = self
2367
return matching
2301
2368
2302
2369
def _ignore_setting_bzrdir(self, format):
2303
2370
pass
2359
2426
class RepositoryFormatKnitPack6RichRoot(RepositoryFormatPack):
2360
2427
"""A repository with rich roots, no subtrees, stacking and btree indexes.
2361
2428
2362
This format should be retained until the second release after bzr 1.7.
2363
2364
1.6.1-subtree[as it might have been] with B+Tree indices.
2429
1.6-rich-root with B+Tree indices.
2365
2430
"""
2366
2431
2367
2432
repository_class = KnitPackRepository
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0