152
151
texts/deltas (via (fileid, revisionid) tuples).
153
152
:param signature_index: A GraphIndex for determining what signatures are
154
153
present in the Pack and accessing the locations of their texts.
155
:param chk_index: A GraphIndex for accessing content by CHK, if the
158
155
self.revision_index = revision_index
159
156
self.inventory_index = inventory_index
160
157
self.text_index = text_index
161
158
self.signature_index = signature_index
162
self.chk_index = chk_index
164
160
def access_tuple(self):
165
161
"""Return a tuple (transport, name) for the pack content."""
226
222
return self.index_name('text', name)
228
224
def _replace_index_with_readonly(self, index_type):
229
unlimited_cache = False
230
if index_type == 'chk':
231
unlimited_cache = True
232
225
setattr(self, index_type + '_index',
233
226
self.index_class(self.index_transport,
234
227
self.index_name(index_type, self.name),
235
self.index_sizes[self.index_offset(index_type)],
236
unlimited_cache=unlimited_cache))
228
self.index_sizes[self.index_offset(index_type)]))
239
231
class ExistingPack(Pack):
240
232
"""An in memory proxy for an existing .pack and its disk indices."""
242
234
def __init__(self, pack_transport, name, revision_index, inventory_index,
243
text_index, signature_index, chk_index=None):
235
text_index, signature_index):
244
236
"""Create an ExistingPack object.
246
238
:param pack_transport: The transport where the pack file resides.
247
239
:param name: The name of the pack on disk in the pack_transport.
249
241
Pack.__init__(self, revision_index, inventory_index, text_index,
250
signature_index, chk_index)
252
244
self.pack_transport = pack_transport
253
245
if None in (revision_index, inventory_index, text_index,
271
263
def __init__(self, name, revision_index, inventory_index, text_index,
272
264
signature_index, upload_transport, pack_transport, index_transport,
273
pack_collection, chk_index=None):
274
266
"""Create a ResumedPack object."""
275
267
ExistingPack.__init__(self, pack_transport, name, revision_index,
276
inventory_index, text_index, signature_index,
268
inventory_index, text_index, signature_index)
278
269
self.upload_transport = upload_transport
279
270
self.index_transport = index_transport
280
271
self.index_sizes = [None, None, None, None]
307
295
self.upload_transport.delete(self.file_name())
308
296
indices = [self.revision_index, self.inventory_index, self.text_index,
309
297
self.signature_index]
310
if self.chk_index is not None:
311
indices.append(self.chk_index)
312
298
for index in indices:
313
299
index._transport.delete(index._name)
315
301
def finish(self):
316
302
self._check_references()
317
index_types = ['revision', 'inventory', 'text', 'signature']
318
if self.chk_index is not None:
319
index_types.append('chk')
320
for index_type in index_types:
303
new_name = '../packs/' + self.file_name()
304
self.upload_transport.rename(self.file_name(), new_name)
305
for index_type in ['revision', 'inventory', 'text', 'signature']:
321
306
old_name = self.index_name(index_type, self.name)
322
307
new_name = '../indices/' + old_name
323
308
self.upload_transport.rename(old_name, new_name)
324
309
self._replace_index_with_readonly(index_type)
325
new_name = '../packs/' + self.file_name()
326
self.upload_transport.rename(self.file_name(), new_name)
327
310
self._state = 'finished'
329
312
def _get_external_refs(self, index):
330
"""Return compression parents for this index that are not present.
332
This returns any compression parents that are referenced by this index,
333
which are not contained *in* this index. They may be present elsewhere.
335
313
return index.external_references(1)
383
355
self._file_mode = file_mode
384
356
# tracks the content written to the .pack file.
385
357
self._hash = osutils.md5()
386
# a tuple with the length in bytes of the indices, once the pack
387
# is finalised. (rev, inv, text, sigs, chk_if_in_use)
358
# a four-tuple with the length in bytes of the indices, once the pack
359
# is finalised. (rev, inv, text, sigs)
388
360
self.index_sizes = None
389
361
# How much data to cache when writing packs. Note that this is not
390
362
# synchronised with reads, because it's not in the transport layer, so
453
423
return bool(self.get_revision_count() or
454
424
self.inventory_index.key_count() or
455
425
self.text_index.key_count() or
456
self.signature_index.key_count() or
457
(self.chk_index is not None and self.chk_index.key_count()))
459
def finish_content(self):
460
if self.name is not None:
464
self._write_data('', flush=True)
465
self.name = self._hash.hexdigest()
426
self.signature_index.key_count())
467
428
def finish(self, suspend=False):
468
429
"""Finish the new pack.
490
454
self._write_index('text', self.text_index, 'file texts', suspend)
491
455
self._write_index('signature', self.signature_index,
492
456
'revision signatures', suspend)
493
if self.chk_index is not None:
494
self.index_sizes.append(None)
495
self._write_index('chk', self.chk_index,
496
'content hash bytes', suspend)
497
457
self.write_stream.close()
498
458
# Note that this will clobber an existing pack with the same name,
499
459
# without checking for hash collisions. While this is undesirable this
587
547
flush_func=flush_func)
588
548
self.add_callback = None
550
def replace_indices(self, index_to_pack, indices):
551
"""Replace the current mappings with fresh ones.
553
This should probably not be used eventually, rather incremental add and
554
removal of indices. It has been added during refactoring of existing
557
:param index_to_pack: A mapping from index objects to
558
(transport, name) tuples for the pack file data.
559
:param indices: A list of indices.
561
# refresh the revision pack map dict without replacing the instance.
562
self.index_to_pack.clear()
563
self.index_to_pack.update(index_to_pack)
564
# XXX: API break - clearly a 'replace' method would be good?
565
self.combined_index._indices[:] = indices
566
# the current add nodes callback for the current writable index if
568
self.add_callback = None
590
570
def add_index(self, index, pack):
591
571
"""Add index to the aggregate, which is an index for Pack pack.
625
605
self.data_access.set_writer(None, None, (None, None))
626
606
self.index_to_pack.clear()
627
607
del self.combined_index._indices[:]
628
del self.combined_index._index_names[:]
629
608
self.add_callback = None
631
def remove_index(self, index):
610
def remove_index(self, index, pack):
632
611
"""Remove index from the indices used to answer queries.
634
613
:param index: An index from the pack parameter.
614
:param pack: A Pack instance.
636
616
del self.index_to_pack[index]
637
pos = self.combined_index._indices.index(index)
638
del self.combined_index._indices[pos]
639
del self.combined_index._index_names[pos]
617
self.combined_index._indices.remove(index)
640
618
if (self.add_callback is not None and
641
619
getattr(index, 'add_nodes', None) == self.add_callback):
642
620
self.add_callback = None
920
897
time.ctime(), self._pack_collection._upload_transport.base, new_pack.random_name,
921
898
new_pack.signature_index.key_count(),
922
899
time.time() - new_pack.start_time)
924
# NB XXX: how to check CHK references are present? perhaps by yielding
925
# the items? How should that interact with stacked repos?
926
if new_pack.chk_index is not None:
928
if 'pack' in debug.debug_flags:
929
mutter('%s: create_pack: chk content copied: %s%s %d items t+%6.3fs',
930
time.ctime(), self._pack_collection._upload_transport.base,
931
new_pack.random_name,
932
new_pack.chk_index.key_count(),
933
time.time() - new_pack.start_time)
934
900
new_pack._check_references()
935
901
if not self._use_pack(new_pack):
940
906
self._pack_collection.allocate(new_pack)
943
def _copy_chks(self, refs=None):
944
# XXX: Todo, recursive follow-pointers facility when fetching some
946
chk_index_map, chk_indices = self._pack_map_and_index_list(
948
chk_nodes = self._index_contents(chk_indices, refs)
950
# TODO: This isn't strictly tasteful as we are accessing some private
951
# variables (_serializer). Perhaps a better way would be to have
952
# Repository._deserialise_chk_node()
953
search_key_func = chk_map.search_key_registry.get(
954
self._pack_collection.repo._serializer.search_key_name)
955
def accumlate_refs(lines):
956
# XXX: move to a generic location
958
bytes = ''.join(lines)
959
node = chk_map._deserialise(bytes, ("unknown",), search_key_func)
960
new_refs.update(node.refs())
961
self._copy_nodes(chk_nodes, chk_index_map, self.new_pack._writer,
962
self.new_pack.chk_index, output_lines=accumlate_refs)
965
def _copy_nodes(self, nodes, index_map, writer, write_index,
967
"""Copy knit nodes between packs with no graph references.
969
:param output_lines: Output full texts of copied items.
909
def _copy_nodes(self, nodes, index_map, writer, write_index):
910
"""Copy knit nodes between packs with no graph references."""
971
911
pb = ui.ui_factory.nested_progress_bar()
973
913
return self._do_copy_nodes(nodes, index_map, writer,
974
write_index, pb, output_lines=output_lines)
978
def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb,
918
def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb):
980
919
# for record verification
981
920
knit = KnitVersionedFiles(None, None)
982
921
# plan a readv on each source pack:
1016
955
izip(reader.iter_records(), pack_readv_requests):
1017
956
raw_data = read_func(None)
1018
957
# check the header only
1019
if output_lines is not None:
1020
output_lines(knit._parse_record(key[-1], raw_data)[0])
1022
df, _ = knit._parse_record_header(key, raw_data)
958
df, _ = knit._parse_record_header(key, raw_data)
1024
960
pos, size = writer.add_bytes_record(raw_data, names)
1025
961
write_index.add_node(key, eol_flag + "%d %d" % (pos, size))
1026
962
pb.update("Copied record", record_index)
1356
1292
:ivar _names: map of {pack_name: (index_size,)}
1359
pack_factory = NewPack
1360
resumed_pack_factory = ResumedPack
1362
1295
def __init__(self, repo, transport, index_transport, upload_transport,
1363
pack_transport, index_builder_class, index_class,
1296
pack_transport, index_builder_class, index_class):
1365
1297
"""Create a new RepositoryPackCollection.
1367
1299
:param transport: Addresses the repository base directory
1398
1328
self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1399
1329
self.text_index = AggregateIndex(self.reload_pack_names, flush)
1400
1330
self.signature_index = AggregateIndex(self.reload_pack_names, flush)
1401
all_indices = [self.revision_index, self.inventory_index,
1402
self.text_index, self.signature_index]
1404
self.chk_index = AggregateIndex(self.reload_pack_names, flush)
1405
all_indices.append(self.chk_index)
1407
# used to determine if we're using a chk_index elsewhere.
1408
self.chk_index = None
1409
# Tell all the CombinedGraphIndex objects about each other, so they can
1410
# share hints about which pack names to search first.
1411
all_combined = [agg_idx.combined_index for agg_idx in all_indices]
1412
for combined_idx in all_combined:
1413
combined_idx.set_sibling_indices(
1414
set(all_combined).difference([combined_idx]))
1415
1331
# resumed packs
1416
1332
self._resumed_packs = []
1419
return '%s(%r)' % (self.__class__.__name__, self.repo)
1421
1334
def add_pack_to_memory(self, pack):
1422
1335
"""Make a Pack object available to the repository to satisfy queries.
1504
1417
'containing %d revisions. Packing %d files into %d affecting %d'
1505
1418
' revisions', self, total_packs, total_revisions, num_old_packs,
1506
1419
num_new_packs, num_revs_affected)
1507
result = self._execute_pack_operations(pack_operations,
1420
self._execute_pack_operations(pack_operations,
1508
1421
reload_func=self._restart_autopack)
1509
mutter('Auto-packing repository %s completed', self)
1512
1424
def _execute_pack_operations(self, pack_operations, _packer_class=Packer,
1513
1425
reload_func=None):
1537
1449
self._remove_pack_from_memory(pack)
1538
1450
# record the newly available packs and stop advertising the old
1540
to_be_obsoleted = []
1541
for _, packs in pack_operations:
1542
to_be_obsoleted.extend(packs)
1543
result = self._save_pack_names(clear_obsolete_packs=True,
1544
obsolete_packs=to_be_obsoleted)
1452
self._save_pack_names(clear_obsolete_packs=True)
1453
# Move the old packs out of the way now they are no longer referenced.
1454
for revision_count, packs in pack_operations:
1455
self._obsolete_packs(packs)
1547
1457
def _flush_new_pack(self):
1548
1458
if self._new_pack is not None:
1557
1467
self.repo.control_files.lock_write()
1559
def _already_packed(self):
1560
"""Is the collection already packed?"""
1561
return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1563
def pack(self, hint=None, clean_obsolete_packs=False):
1564
1470
"""Pack the pack collection totally."""
1565
1471
self.ensure_loaded()
1566
1472
total_packs = len(self._names)
1567
if self._already_packed():
1474
# This is arguably wrong because we might not be optimal, but for
1475
# now lets leave it in. (e.g. reconcile -> one pack. But not
1569
1478
total_revisions = self.revision_index.combined_index.key_count()
1570
1479
# XXX: the following may want to be a class, to pack with a given
1572
1481
mutter('Packing repository %s, which has %d pack files, '
1573
'containing %d revisions with hint %r.', self, total_packs,
1574
total_revisions, hint)
1482
'containing %d revisions into 1 packs.', self, total_packs,
1575
1484
# determine which packs need changing
1485
pack_distribution = [1]
1576
1486
pack_operations = [[0, []]]
1577
1487
for pack in self.all_packs():
1578
if hint is None or pack.name in hint:
1579
# Either no hint was provided (so we are packing everything),
1580
# or this pack was included in the hint.
1581
pack_operations[-1][0] += pack.get_revision_count()
1582
pack_operations[-1][1].append(pack)
1488
pack_operations[-1][0] += pack.get_revision_count()
1489
pack_operations[-1][1].append(pack)
1583
1490
self._execute_pack_operations(pack_operations, OptimisingPacker)
1585
if clean_obsolete_packs:
1586
self._clear_obsolete_packs()
1588
1492
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1589
1493
"""Plan a pack operation.
1677
1581
inv_index = self._make_index(name, '.iix')
1678
1582
txt_index = self._make_index(name, '.tix')
1679
1583
sig_index = self._make_index(name, '.six')
1680
if self.chk_index is not None:
1681
chk_index = self._make_index(name, '.cix', unlimited_cache=True)
1684
1584
result = ExistingPack(self._pack_transport, name, rev_index,
1685
inv_index, txt_index, sig_index, chk_index)
1585
inv_index, txt_index, sig_index)
1686
1586
self.add_pack_to_memory(result)
1702
1602
inv_index = self._make_index(name, '.iix', resume=True)
1703
1603
txt_index = self._make_index(name, '.tix', resume=True)
1704
1604
sig_index = self._make_index(name, '.six', resume=True)
1705
if self.chk_index is not None:
1706
chk_index = self._make_index(name, '.cix', resume=True,
1707
unlimited_cache=True)
1710
result = self.resumed_pack_factory(name, rev_index, inv_index,
1711
txt_index, sig_index, self._upload_transport,
1712
self._pack_transport, self._index_transport, self,
1713
chk_index=chk_index)
1605
result = ResumedPack(name, rev_index, inv_index, txt_index,
1606
sig_index, self._upload_transport, self._pack_transport,
1607
self._index_transport, self)
1714
1608
except errors.NoSuchFile, e:
1715
1609
raise errors.UnresumableWriteGroup(self.repo, [name], str(e))
1716
1610
self.add_pack_to_memory(result)
1784
1677
:param return: None.
1786
1679
for pack in packs:
1788
pack.pack_transport.rename(pack.file_name(),
1789
'../obsolete_packs/' + pack.file_name())
1790
except (errors.PathError, errors.TransportError), e:
1791
# TODO: Should these be warnings or mutters?
1792
mutter("couldn't rename obsolete pack, skipping it:\n%s"
1680
pack.pack_transport.rename(pack.file_name(),
1681
'../obsolete_packs/' + pack.file_name())
1794
1682
# TODO: Probably needs to know all possible indices for this pack
1795
1683
# - or maybe list the directory and move all indices matching this
1796
1684
# name whether we recognize it or not?
1797
suffixes = ['.iix', '.six', '.tix', '.rix']
1798
if self.chk_index is not None:
1799
suffixes.append('.cix')
1800
for suffix in suffixes:
1802
self._index_transport.rename(pack.name + suffix,
1803
'../obsolete_packs/' + pack.name + suffix)
1804
except (errors.PathError, errors.TransportError), e:
1805
mutter("couldn't rename obsolete index, skipping it:\n%s"
1685
for suffix in ('.iix', '.six', '.tix', '.rix'):
1686
self._index_transport.rename(pack.name + suffix,
1687
'../obsolete_packs/' + pack.name + suffix)
1808
1689
def pack_distribution(self, total_revisions):
1809
1690
"""Generate a list of the number of revisions to put in each pack.
1835
1716
self._remove_pack_indices(pack)
1836
1717
self.packs.remove(pack)
1838
def _remove_pack_indices(self, pack, ignore_missing=False):
1839
"""Remove the indices for pack from the aggregated indices.
1841
:param ignore_missing: Suppress KeyErrors from calling remove_index.
1843
for index_type in Pack.index_definitions.keys():
1844
attr_name = index_type + '_index'
1845
aggregate_index = getattr(self, attr_name)
1846
if aggregate_index is not None:
1847
pack_index = getattr(pack, attr_name)
1849
aggregate_index.remove_index(pack_index)
1719
def _remove_pack_indices(self, pack):
1720
"""Remove the indices for pack from the aggregated indices."""
1721
self.revision_index.remove_index(pack.revision_index, pack)
1722
self.inventory_index.remove_index(pack.inventory_index, pack)
1723
self.text_index.remove_index(pack.text_index, pack)
1724
self.signature_index.remove_index(pack.signature_index, pack)
1855
1726
def reset(self):
1856
1727
"""Clear all cached data."""
1857
1728
# cached revision data
1729
self.repo._revision_knit = None
1858
1730
self.revision_index.clear()
1859
1731
# cached signature data
1732
self.repo._signature_knit = None
1860
1733
self.signature_index.clear()
1861
1734
# cached file text data
1862
1735
self.text_index.clear()
1736
self.repo._text_knit = None
1863
1737
# cached inventory data
1864
1738
self.inventory_index.clear()
1866
if self.chk_index is not None:
1867
self.chk_index.clear()
1868
1739
# remove the open pack
1869
1740
self._new_pack = None
1870
1741
# information about packs.
1965
1835
:param clear_obsolete_packs: If True, clear out the contents of the
1966
1836
obsolete_packs directory.
1967
:param obsolete_packs: Packs that are obsolete once the new pack-names
1968
file has been written.
1969
:return: A list of the names saved that were not previously on disk.
1971
already_obsolete = []
1972
1838
self.lock_names()
1974
1840
builder = self._index_builder_class()
1975
(disk_nodes, deleted_nodes, new_nodes,
1976
orig_disk_nodes) = self._diff_pack_names()
1841
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1977
1842
# TODO: handle same-name, index-size-changes here -
1978
1843
# e.g. use the value from disk, not ours, *unless* we're the one
1981
1846
builder.add_node(key, value)
1982
1847
self.transport.put_file('pack-names', builder.finish(),
1983
1848
mode=self.repo.bzrdir._get_file_mode())
1849
# move the baseline forward
1984
1850
self._packs_at_load = disk_nodes
1985
1851
if clear_obsolete_packs:
1988
to_preserve = set([o.name for o in obsolete_packs])
1989
already_obsolete = self._clear_obsolete_packs(to_preserve)
1852
self._clear_obsolete_packs()
1991
1854
self._unlock_names()
1992
1855
# synchronise the memory packs list with what we just wrote:
1993
1856
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1995
# TODO: We could add one more condition here. "if o.name not in
1996
# orig_disk_nodes and o != the new_pack we haven't written to
1997
# disk yet. However, the new pack object is not easily
1998
# accessible here (it would have to be passed through the
1999
# autopacking code, etc.)
2000
obsolete_packs = [o for o in obsolete_packs
2001
if o.name not in already_obsolete]
2002
self._obsolete_packs(obsolete_packs)
2003
return [new_node[0][0] for new_node in new_nodes]
2005
1858
def reload_pack_names(self):
2006
1859
"""Sync our pack listing with what is present in the repository.
2022
1875
# out the new value.
2023
(disk_nodes, deleted_nodes, new_nodes,
2024
orig_disk_nodes) = self._diff_pack_names()
2025
# _packs_at_load is meant to be the explicit list of names in
2026
# 'pack-names' at then start. As such, it should not contain any
2027
# pending names that haven't been written out yet.
2028
self._packs_at_load = orig_disk_nodes
1876
disk_nodes, _, _ = self._diff_pack_names()
1877
self._packs_at_load = disk_nodes
2029
1878
(removed, added,
2030
1879
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
2031
1880
if removed or added or modified:
2041
1890
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
2043
def _clear_obsolete_packs(self, preserve=None):
1892
def _clear_obsolete_packs(self):
2044
1893
"""Delete everything from the obsolete-packs directory.
2046
:return: A list of pack identifiers (the filename without '.pack') that
2047
were found in obsolete_packs.
2050
1895
obsolete_pack_transport = self.transport.clone('obsolete_packs')
2051
if preserve is None:
2053
1896
for filename in obsolete_pack_transport.list_dir('.'):
2054
name, ext = osutils.splitext(filename)
2057
if name in preserve:
2060
1898
obsolete_pack_transport.delete(filename)
2061
1899
except (errors.PathError, errors.TransportError), e:
2062
warning("couldn't delete obsolete pack, skipping it:\n%s"
1900
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
2066
1902
def _start_write_group(self):
2067
1903
# Do not permit preparation for writing if we're not in a 'write lock'.
2068
1904
if not self.repo.is_write_locked():
2069
1905
raise errors.NotWriteLocked(self)
2070
self._new_pack = self.pack_factory(self, upload_suffix='.pack',
1906
self._new_pack = NewPack(self, upload_suffix='.pack',
2071
1907
file_mode=self.repo.bzrdir._get_file_mode())
2072
1908
# allow writing: queue writes to a new index
2073
1909
self.revision_index.add_writable_index(self._new_pack.revision_index,
2076
1912
self._new_pack)
2077
1913
self.text_index.add_writable_index(self._new_pack.text_index,
2078
1914
self._new_pack)
2079
self._new_pack.text_index.set_optimize(combine_backing_indices=False)
2080
1915
self.signature_index.add_writable_index(self._new_pack.signature_index,
2081
1916
self._new_pack)
2082
if self.chk_index is not None:
2083
self.chk_index.add_writable_index(self._new_pack.chk_index,
2085
self.repo.chk_bytes._index._add_callback = self.chk_index.add_callback
2086
self._new_pack.chk_index.set_optimize(combine_backing_indices=False)
2088
1918
self.repo.inventories._index._add_callback = self.inventory_index.add_callback
2089
1919
self.repo.revisions._index._add_callback = self.revision_index.add_callback
2094
1924
# FIXME: just drop the transient index.
2095
1925
# forget what names there are
2096
1926
if self._new_pack is not None:
2097
operation = cleanup.OperationWithCleanups(self._new_pack.abort)
2098
operation.add_cleanup(setattr, self, '_new_pack', None)
2099
# If we aborted while in the middle of finishing the write
2100
# group, _remove_pack_indices could fail because the indexes are
2101
# already gone. But they're not there we shouldn't fail in this
2102
# case, so we pass ignore_missing=True.
2103
operation.add_cleanup(self._remove_pack_indices, self._new_pack,
2104
ignore_missing=True)
2105
operation.run_simple()
1928
self._new_pack.abort()
1930
# XXX: If we aborted while in the middle of finishing the write
1931
# group, _remove_pack_indices can fail because the indexes are
1932
# already gone. If they're not there we shouldn't fail in this
1933
# case. -- mbp 20081113
1934
self._remove_pack_indices(self._new_pack)
1935
self._new_pack = None
2106
1936
for resumed_pack in self._resumed_packs:
2107
operation = cleanup.OperationWithCleanups(resumed_pack.abort)
2108
# See comment in previous finally block.
2109
operation.add_cleanup(self._remove_pack_indices, resumed_pack,
2110
ignore_missing=True)
2111
operation.run_simple()
1938
resumed_pack.abort()
1940
# See comment in previous finally block.
1942
self._remove_pack_indices(resumed_pack)
2112
1945
del self._resumed_packs[:]
1946
self.repo._text_knit = None
2114
1948
def _remove_resumed_pack_indices(self):
2115
1949
for resumed_pack in self._resumed_packs:
2116
1950
self._remove_pack_indices(resumed_pack)
2117
1951
del self._resumed_packs[:]
2119
def _check_new_inventories(self):
2120
"""Detect missing inventories in this write group.
2122
:returns: list of strs, summarising any problems found. If the list is
2123
empty no problems were found.
2125
# The base implementation does no checks. GCRepositoryPackCollection
2129
1953
def _commit_write_group(self):
2130
1954
all_missing = set()
2131
1955
for prefix, versioned_file in (
2140
1964
raise errors.BzrCheckError(
2141
1965
"Repository %s has missing compression parent(s) %r "
2142
1966
% (self.repo, sorted(all_missing)))
2143
problems = self._check_new_inventories()
2145
problems_summary = '\n'.join(problems)
2146
raise errors.BzrCheckError(
2147
"Cannot add revision(s) to repository: " + problems_summary)
2148
1967
self._remove_pack_indices(self._new_pack)
2149
any_new_content = False
1968
should_autopack = False
2150
1969
if self._new_pack.data_inserted():
2151
1970
# get all the data to disk and read to use
2152
1971
self._new_pack.finish()
2153
1972
self.allocate(self._new_pack)
2154
1973
self._new_pack = None
2155
any_new_content = True
1974
should_autopack = True
2157
1976
self._new_pack.abort()
2158
1977
self._new_pack = None
2240
2056
self.revisions = KnitVersionedFiles(
2241
2057
_KnitGraphIndex(self._pack_collection.revision_index.combined_index,
2242
2058
add_callback=self._pack_collection.revision_index.add_callback,
2243
deltas=False, parents=True, is_locked=self.is_locked,
2244
track_external_parent_refs=True),
2059
deltas=False, parents=True, is_locked=self.is_locked),
2245
2060
data_access=self._pack_collection.revision_index.data_access,
2246
2061
max_delta_chain=0)
2247
2062
self.signatures = KnitVersionedFiles(
2256
2071
deltas=True, parents=True, is_locked=self.is_locked),
2257
2072
data_access=self._pack_collection.text_index.data_access,
2258
2073
max_delta_chain=200)
2259
if _format.supports_chks:
2260
# No graph, no compression:- references from chks are between
2261
# different objects not temporal versions of the same; and without
2262
# some sort of temporal structure knit compression will just fail.
2263
self.chk_bytes = KnitVersionedFiles(
2264
_KnitGraphIndex(self._pack_collection.chk_index.combined_index,
2265
add_callback=self._pack_collection.chk_index.add_callback,
2266
deltas=False, parents=False, is_locked=self.is_locked),
2267
data_access=self._pack_collection.chk_index.data_access,
2270
self.chk_bytes = None
2271
2074
# True when the repository object is 'write locked' (as opposed to the
2272
2075
# physical lock only taken out around changes to the pack-names list.)
2273
2076
# Another way to represent this would be a decorator around the control
2280
2083
self._reconcile_fixes_text_parents = True
2281
2084
self._reconcile_backsup_inventory = False
2283
def _warn_if_deprecated(self, branch=None):
2086
def _warn_if_deprecated(self):
2284
2087
# This class isn't deprecated, but one sub-format is
2285
2088
if isinstance(self._format, RepositoryFormatKnitPack5RichRootBroken):
2286
super(KnitPackRepository, self)._warn_if_deprecated(branch)
2089
from bzrlib import repository
2090
if repository._deprecation_warning_done:
2092
repository._deprecation_warning_done = True
2093
warning("Format %s for %s is deprecated - please use"
2094
" 'bzr upgrade --1.6.1-rich-root'"
2095
% (self._format, self.bzrdir.transport.base))
2288
2097
def _abort_write_group(self):
2289
self.revisions._index._key_dependencies.clear()
2290
2098
self._pack_collection._abort_write_group()
2292
def _get_source(self, to_format):
2293
if to_format.network_name() == self._format.network_name():
2294
return KnitPackStreamSource(self, to_format)
2295
return super(KnitPackRepository, self)._get_source(to_format)
2100
def _find_inconsistent_revision_parents(self):
2101
"""Find revisions with incorrectly cached parents.
2103
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
2104
parents-in-revision).
2106
if not self.is_locked():
2107
raise errors.ObjectNotLocked(self)
2108
pb = ui.ui_factory.nested_progress_bar()
2111
revision_nodes = self._pack_collection.revision_index \
2112
.combined_index.iter_all_entries()
2113
index_positions = []
2114
# Get the cached index values for all revisions, and also the location
2115
# in each index of the revision text so we can perform linear IO.
2116
for index, key, value, refs in revision_nodes:
2117
pos, length = value[1:].split(' ')
2118
index_positions.append((index, int(pos), key[0],
2119
tuple(parent[0] for parent in refs[0])))
2120
pb.update("Reading revision index", 0, 0)
2121
index_positions.sort()
2122
batch_count = len(index_positions) / 1000 + 1
2123
pb.update("Checking cached revision graph", 0, batch_count)
2124
for offset in xrange(batch_count):
2125
pb.update("Checking cached revision graph", offset)
2126
to_query = index_positions[offset * 1000:(offset + 1) * 1000]
2129
rev_ids = [item[2] for item in to_query]
2130
revs = self.get_revisions(rev_ids)
2131
for revision, item in zip(revs, to_query):
2132
index_parents = item[3]
2133
rev_parents = tuple(revision.parent_ids)
2134
if index_parents != rev_parents:
2135
result.append((revision.revision_id, index_parents, rev_parents))
2297
2140
def _make_parents_provider(self):
2298
2141
return graph.CachingParentsProvider(self)
2306
2149
self._pack_collection._start_write_group()
2308
2151
def _commit_write_group(self):
2309
hint = self._pack_collection._commit_write_group()
2310
self.revisions._index._key_dependencies.clear()
2152
return self._pack_collection._commit_write_group()
2313
2154
def suspend_write_group(self):
2314
2155
# XXX check self._write_group is self.get_transaction()?
2315
2156
tokens = self._pack_collection._suspend_write_group()
2316
self.revisions._index._key_dependencies.clear()
2317
2157
self._write_group = None
2320
2160
def _resume_write_group(self, tokens):
2321
2161
self._start_write_group()
2323
self._pack_collection._resume_write_group(tokens)
2324
except errors.UnresumableWriteGroup:
2325
self._abort_write_group()
2327
for pack in self._pack_collection._resumed_packs:
2328
self.revisions._index.scan_unvalidated_index(pack.revision_index)
2162
self._pack_collection._resume_write_group(tokens)
2330
2164
def get_transaction(self):
2331
2165
if self._write_lock_count:
2413
2237
transaction = self._transaction
2414
2238
self._transaction = None
2415
2239
transaction.finish()
2240
for repo in self._fallback_repositories:
2417
2243
self.control_files.unlock()
2419
if not self.is_locked():
2420
2244
for repo in self._fallback_repositories:
2424
class KnitPackStreamSource(StreamSource):
2425
"""A StreamSource used to transfer data between same-format KnitPack repos.
2427
This source assumes:
2428
1) Same serialization format for all objects
2429
2) Same root information
2430
3) XML format inventories
2431
4) Atomic inserts (so we can stream inventory texts before text
2436
def __init__(self, from_repository, to_format):
2437
super(KnitPackStreamSource, self).__init__(from_repository, to_format)
2438
self._text_keys = None
2439
self._text_fetch_order = 'unordered'
2441
def _get_filtered_inv_stream(self, revision_ids):
2442
from_repo = self.from_repository
2443
parent_ids = from_repo._find_parent_ids_of_revisions(revision_ids)
2444
parent_keys = [(p,) for p in parent_ids]
2445
find_text_keys = from_repo._find_text_key_references_from_xml_inventory_lines
2446
parent_text_keys = set(find_text_keys(
2447
from_repo._inventory_xml_lines_for_keys(parent_keys)))
2448
content_text_keys = set()
2449
knit = KnitVersionedFiles(None, None)
2450
factory = KnitPlainFactory()
2451
def find_text_keys_from_content(record):
2452
if record.storage_kind not in ('knit-delta-gz', 'knit-ft-gz'):
2453
raise ValueError("Unknown content storage kind for"
2454
" inventory text: %s" % (record.storage_kind,))
2455
# It's a knit record, it has a _raw_record field (even if it was
2456
# reconstituted from a network stream).
2457
raw_data = record._raw_record
2458
# read the entire thing
2459
revision_id = record.key[-1]
2460
content, _ = knit._parse_record(revision_id, raw_data)
2461
if record.storage_kind == 'knit-delta-gz':
2462
line_iterator = factory.get_linedelta_content(content)
2463
elif record.storage_kind == 'knit-ft-gz':
2464
line_iterator = factory.get_fulltext_content(content)
2465
content_text_keys.update(find_text_keys(
2466
[(line, revision_id) for line in line_iterator]))
2467
revision_keys = [(r,) for r in revision_ids]
2468
def _filtered_inv_stream():
2469
source_vf = from_repo.inventories
2470
stream = source_vf.get_record_stream(revision_keys,
2472
for record in stream:
2473
if record.storage_kind == 'absent':
2474
raise errors.NoSuchRevision(from_repo, record.key)
2475
find_text_keys_from_content(record)
2477
self._text_keys = content_text_keys - parent_text_keys
2478
return ('inventories', _filtered_inv_stream())
2480
def _get_text_stream(self):
2481
# Note: We know we don't have to handle adding root keys, because both
2482
# the source and target are the identical network name.
2483
text_stream = self.from_repository.texts.get_record_stream(
2484
self._text_keys, self._text_fetch_order, False)
2485
return ('texts', text_stream)
2487
def get_stream(self, search):
2488
revision_ids = search.get_keys()
2489
for stream_info in self._fetch_revision_texts(revision_ids):
2491
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
2492
yield self._get_filtered_inv_stream(revision_ids)
2493
yield self._get_text_stream()
2497
2248
class RepositoryFormatPack(MetaDirRepositoryFormat):
2498
2249
"""Format logic for pack structured repositories.
2546
2295
utf8_files = [('format', self.get_format_string())]
2548
2297
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2549
repository = self.open(a_bzrdir=a_bzrdir, _found=True)
2550
self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
2298
return self.open(a_bzrdir=a_bzrdir, _found=True)
2553
2300
def open(self, a_bzrdir, _found=False, _override_transport=None):
2554
2301
"""See RepositoryFormat.open().
2636
2385
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2387
def check_conversion_target(self, target_format):
2388
if not target_format.rich_root_data:
2389
raise errors.BadConversionTarget(
2390
'Does not support rich root data.', target_format)
2391
if not getattr(target_format, 'supports_tree_reference', False):
2392
raise errors.BadConversionTarget(
2393
'Does not support nested trees', target_format)
2638
2395
def get_format_string(self):
2639
2396
"""See RepositoryFormat.get_format_string()."""
2640
2397
return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
2877
2660
return "Packs 6 rich-root (uses btree indexes, requires bzr 1.9)"
2663
class RepositoryFormatPackDevelopment2(RepositoryFormatPack):
2664
"""A no-subtrees development repository.
2666
This format should be retained until the second release after bzr 1.7.
2668
This is pack-1.6.1 with B+Tree indices.
2671
repository_class = KnitPackRepository
2672
_commit_builder_class = PackCommitBuilder
2673
supports_external_lookups = True
2674
# What index classes to use
2675
index_builder_class = BTreeBuilder
2676
index_class = BTreeGraphIndex
2677
# Set to true to get the fast-commit code path tested until a really fast
2678
# format lands in trunk. Not actually fast in this format.
2682
def _serializer(self):
2683
return xml5.serializer_v5
2685
def _get_matching_bzrdir(self):
2686
return bzrdir.format_registry.make_bzrdir('development2')
2688
def _ignore_setting_bzrdir(self, format):
2691
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2693
def get_format_string(self):
2694
"""See RepositoryFormat.get_format_string()."""
2695
return "Bazaar development format 2 (needs bzr.dev from before 1.8)\n"
2697
def get_format_description(self):
2698
"""See RepositoryFormat.get_format_description()."""
2699
return ("Development repository format, currently the same as "
2700
"1.6.1 with B+Trees.\n")
2702
def check_conversion_target(self, target_format):
2880
2706
class RepositoryFormatPackDevelopment2Subtree(RepositoryFormatPack):
2881
2707
"""A subtrees development repository.
2883
2709
This format should be retained until the second release after bzr 1.7.
2885
2711
1.6.1-subtree[as it might have been] with B+Tree indices.
2887
This is [now] retained until we have a CHK based subtree format in
2891
2714
repository_class = KnitPackRepository
2892
2715
_commit_builder_class = PackRootCommitBuilder
2893
2716
rich_root_data = True
2895
2717
supports_tree_reference = True
2896
2718
supports_external_lookups = True
2897
2719
# What index classes to use
2905
2727
def _get_matching_bzrdir(self):
2906
2728
return bzrdir.format_registry.make_bzrdir(
2907
'development-subtree')
2729
'development2-subtree')
2909
2731
def _ignore_setting_bzrdir(self, format):
2912
2734
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2736
def check_conversion_target(self, target_format):
2737
if not target_format.rich_root_data:
2738
raise errors.BadConversionTarget(
2739
'Does not support rich root data.', target_format)
2740
if not getattr(target_format, 'supports_tree_reference', False):
2741
raise errors.BadConversionTarget(
2742
'Does not support nested trees', target_format)
2914
2744
def get_format_string(self):
2915
2745
"""See RepositoryFormat.get_format_string()."""
2916
2746
return ("Bazaar development format 2 with subtree support "
added
removed
bzrlib/repofmt/pack_repo.py
