227
224
return self.index_name('text', name)
229
226
def _replace_index_with_readonly(self, index_type):
230
unlimited_cache = False
231
if index_type == 'chk':
232
unlimited_cache = True
233
227
setattr(self, index_type + '_index',
234
228
self.index_class(self.index_transport,
235
229
self.index_name(index_type, self.name),
236
self.index_sizes[self.index_offset(index_type)],
237
unlimited_cache=unlimited_cache))
230
self.index_sizes[self.index_offset(index_type)]))
240
233
class ExistingPack(Pack):
588
574
flush_func=flush_func)
589
575
self.add_callback = None
577
def replace_indices(self, index_to_pack, indices):
578
"""Replace the current mappings with fresh ones.
580
This should probably not be used eventually, rather incremental add and
581
removal of indices. It has been added during refactoring of existing
584
:param index_to_pack: A mapping from index objects to
585
(transport, name) tuples for the pack file data.
586
:param indices: A list of indices.
588
# refresh the revision pack map dict without replacing the instance.
589
self.index_to_pack.clear()
590
self.index_to_pack.update(index_to_pack)
591
# XXX: API break - clearly a 'replace' method would be good?
592
self.combined_index._indices[:] = indices
593
# the current add nodes callback for the current writable index if
595
self.add_callback = None
591
597
def add_index(self, index, pack):
592
598
"""Add index to the aggregate, which is an index for Pack pack.
600
606
# expose it to the index map
601
607
self.index_to_pack[index] = pack.access_tuple()
602
608
# put it at the front of the linear index list
603
self.combined_index.insert_index(0, index, pack.name)
609
self.combined_index.insert_index(0, index)
605
611
def add_writable_index(self, index, pack):
606
612
"""Add an index which is able to have data added to it.
626
632
self.data_access.set_writer(None, None, (None, None))
627
633
self.index_to_pack.clear()
628
634
del self.combined_index._indices[:]
629
del self.combined_index._index_names[:]
630
635
self.add_callback = None
632
def remove_index(self, index):
637
def remove_index(self, index, pack):
633
638
"""Remove index from the indices used to answer queries.
635
640
:param index: An index from the pack parameter.
641
:param pack: A Pack instance.
637
643
del self.index_to_pack[index]
638
pos = self.combined_index._indices.index(index)
639
del self.combined_index._indices[pos]
640
del self.combined_index._index_names[pos]
644
self.combined_index._indices.remove(index)
641
645
if (self.add_callback is not None and
642
646
getattr(index, 'add_nodes', None) == self.add_callback):
643
647
self.add_callback = None
1101
1105
iterator is a tuple with:
1102
1106
index, readv_vector, node_vector. readv_vector is a list ready to
1103
1107
hand to the transport readv method, and node_vector is a list of
1104
(key, eol_flag, references) for the node retrieved by the
1108
(key, eol_flag, references) for the the node retrieved by the
1105
1109
matching readv_vector.
1107
1111
# group by pack so we do one readv per pack
1399
1403
self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1400
1404
self.text_index = AggregateIndex(self.reload_pack_names, flush)
1401
1405
self.signature_index = AggregateIndex(self.reload_pack_names, flush)
1402
all_indices = [self.revision_index, self.inventory_index,
1403
self.text_index, self.signature_index]
1404
1406
if use_chk_index:
1405
1407
self.chk_index = AggregateIndex(self.reload_pack_names, flush)
1406
all_indices.append(self.chk_index)
1408
1409
# used to determine if we're using a chk_index elsewhere.
1409
1410
self.chk_index = None
1410
# Tell all the CombinedGraphIndex objects about each other, so they can
1411
# share hints about which pack names to search first.
1412
all_combined = [agg_idx.combined_index for agg_idx in all_indices]
1413
for combined_idx in all_combined:
1414
combined_idx.set_sibling_indices(
1415
set(all_combined).difference([combined_idx]))
1416
1411
# resumed packs
1417
1412
self._resumed_packs = []
1420
return '%s(%r)' % (self.__class__.__name__, self.repo)
1422
1414
def add_pack_to_memory(self, pack):
1423
1415
"""Make a Pack object available to the repository to satisfy queries.
1538
1530
self._remove_pack_from_memory(pack)
1539
1531
# record the newly available packs and stop advertising the old
1541
to_be_obsoleted = []
1542
for _, packs in pack_operations:
1543
to_be_obsoleted.extend(packs)
1544
result = self._save_pack_names(clear_obsolete_packs=True,
1545
obsolete_packs=to_be_obsoleted)
1533
result = self._save_pack_names(clear_obsolete_packs=True)
1534
# Move the old packs out of the way now they are no longer referenced.
1535
for revision_count, packs in pack_operations:
1536
self._obsolete_packs(packs)
1548
1539
def _flush_new_pack(self):
1576
1567
# determine which packs need changing
1577
1568
pack_operations = [[0, []]]
1578
1569
for pack in self.all_packs():
1579
if hint is None or pack.name in hint:
1580
# Either no hint was provided (so we are packing everything),
1581
# or this pack was included in the hint.
1570
if not hint or pack.name in hint:
1582
1571
pack_operations[-1][0] += pack.get_revision_count()
1583
1572
pack_operations[-1][1].append(pack)
1584
1573
self._execute_pack_operations(pack_operations, OptimisingPacker)
1586
if clean_obsolete_packs:
1587
self._clear_obsolete_packs()
1589
1575
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1590
1576
"""Plan a pack operation.
1679
1665
txt_index = self._make_index(name, '.tix')
1680
1666
sig_index = self._make_index(name, '.six')
1681
1667
if self.chk_index is not None:
1682
chk_index = self._make_index(name, '.cix', unlimited_cache=True)
1668
chk_index = self._make_index(name, '.cix')
1684
1670
chk_index = None
1685
1671
result = ExistingPack(self._pack_transport, name, rev_index,
1704
1690
txt_index = self._make_index(name, '.tix', resume=True)
1705
1691
sig_index = self._make_index(name, '.six', resume=True)
1706
1692
if self.chk_index is not None:
1707
chk_index = self._make_index(name, '.cix', resume=True,
1708
unlimited_cache=True)
1693
chk_index = self._make_index(name, '.cix', resume=True)
1710
1695
chk_index = None
1711
1696
result = self.resumed_pack_factory(name, rev_index, inv_index,
1741
1726
return self._index_class(self.transport, 'pack-names', None
1742
1727
).iter_all_entries()
1744
def _make_index(self, name, suffix, resume=False, unlimited_cache=False):
1729
def _make_index(self, name, suffix, resume=False):
1745
1730
size_offset = self._suffix_offsets[suffix]
1746
1731
index_name = name + suffix
1751
1736
transport = self._index_transport
1752
1737
index_size = self._names[name][size_offset]
1753
return self._index_class(transport, index_name, index_size,
1754
unlimited_cache=unlimited_cache)
1738
return self._index_class(transport, index_name, index_size)
1756
1740
def _max_pack_count(self, total_revisions):
1757
1741
"""Return the maximum number of packs to use for total revisions.
1785
1769
:param return: None.
1787
1771
for pack in packs:
1789
pack.pack_transport.rename(pack.file_name(),
1790
'../obsolete_packs/' + pack.file_name())
1791
except (errors.PathError, errors.TransportError), e:
1792
# TODO: Should these be warnings or mutters?
1793
mutter("couldn't rename obsolete pack, skipping it:\n%s"
1772
pack.pack_transport.rename(pack.file_name(),
1773
'../obsolete_packs/' + pack.file_name())
1795
1774
# TODO: Probably needs to know all possible indices for this pack
1796
1775
# - or maybe list the directory and move all indices matching this
1797
1776
# name whether we recognize it or not?
1799
1778
if self.chk_index is not None:
1800
1779
suffixes.append('.cix')
1801
1780
for suffix in suffixes:
1803
self._index_transport.rename(pack.name + suffix,
1804
'../obsolete_packs/' + pack.name + suffix)
1805
except (errors.PathError, errors.TransportError), e:
1806
mutter("couldn't rename obsolete index, skipping it:\n%s"
1781
self._index_transport.rename(pack.name + suffix,
1782
'../obsolete_packs/' + pack.name + suffix)
1809
1784
def pack_distribution(self, total_revisions):
1810
1785
"""Generate a list of the number of revisions to put in each pack.
1836
1811
self._remove_pack_indices(pack)
1837
1812
self.packs.remove(pack)
1839
def _remove_pack_indices(self, pack, ignore_missing=False):
1840
"""Remove the indices for pack from the aggregated indices.
1842
:param ignore_missing: Suppress KeyErrors from calling remove_index.
1844
for index_type in Pack.index_definitions.keys():
1845
attr_name = index_type + '_index'
1846
aggregate_index = getattr(self, attr_name)
1847
if aggregate_index is not None:
1848
pack_index = getattr(pack, attr_name)
1850
aggregate_index.remove_index(pack_index)
1814
def _remove_pack_indices(self, pack):
1815
"""Remove the indices for pack from the aggregated indices."""
1816
self.revision_index.remove_index(pack.revision_index, pack)
1817
self.inventory_index.remove_index(pack.inventory_index, pack)
1818
self.text_index.remove_index(pack.text_index, pack)
1819
self.signature_index.remove_index(pack.signature_index, pack)
1820
if self.chk_index is not None:
1821
self.chk_index.remove_index(pack.chk_index, pack)
1856
1823
def reset(self):
1857
1824
"""Clear all cached data."""
1909
1875
disk_nodes.difference_update(deleted_nodes)
1910
1876
disk_nodes.update(new_nodes)
1912
return disk_nodes, deleted_nodes, new_nodes, orig_disk_nodes
1878
return disk_nodes, deleted_nodes, new_nodes
1914
1880
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1915
1881
"""Given the correct set of pack files, update our saved info.
1966
1932
:param clear_obsolete_packs: If True, clear out the contents of the
1967
1933
obsolete_packs directory.
1968
:param obsolete_packs: Packs that are obsolete once the new pack-names
1969
file has been written.
1970
1934
:return: A list of the names saved that were not previously on disk.
1972
already_obsolete = []
1973
1936
self.lock_names()
1975
1938
builder = self._index_builder_class()
1976
(disk_nodes, deleted_nodes, new_nodes,
1977
orig_disk_nodes) = self._diff_pack_names()
1939
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1978
1940
# TODO: handle same-name, index-size-changes here -
1979
1941
# e.g. use the value from disk, not ours, *unless* we're the one
1982
1944
builder.add_node(key, value)
1983
1945
self.transport.put_file('pack-names', builder.finish(),
1984
1946
mode=self.repo.bzrdir._get_file_mode())
1947
# move the baseline forward
1985
1948
self._packs_at_load = disk_nodes
1986
1949
if clear_obsolete_packs:
1989
to_preserve = set([o.name for o in obsolete_packs])
1990
already_obsolete = self._clear_obsolete_packs(to_preserve)
1950
self._clear_obsolete_packs()
1992
1952
self._unlock_names()
1993
1953
# synchronise the memory packs list with what we just wrote:
1994
1954
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1996
# TODO: We could add one more condition here. "if o.name not in
1997
# orig_disk_nodes and o != the new_pack we haven't written to
1998
# disk yet. However, the new pack object is not easily
1999
# accessible here (it would have to be passed through the
2000
# autopacking code, etc.)
2001
obsolete_packs = [o for o in obsolete_packs
2002
if o.name not in already_obsolete]
2003
self._obsolete_packs(obsolete_packs)
2004
1955
return [new_node[0][0] for new_node in new_nodes]
2006
1957
def reload_pack_names(self):
2023
1974
# out the new value.
2024
(disk_nodes, deleted_nodes, new_nodes,
2025
orig_disk_nodes) = self._diff_pack_names()
2026
# _packs_at_load is meant to be the explicit list of names in
2027
# 'pack-names' at then start. As such, it should not contain any
2028
# pending names that haven't been written out yet.
2029
self._packs_at_load = orig_disk_nodes
1975
disk_nodes, _, _ = self._diff_pack_names()
1976
self._packs_at_load = disk_nodes
2030
1977
(removed, added,
2031
1978
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
2032
1979
if removed or added or modified:
2042
1989
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
2044
def _clear_obsolete_packs(self, preserve=None):
1991
def _clear_obsolete_packs(self):
2045
1992
"""Delete everything from the obsolete-packs directory.
2047
:return: A list of pack identifiers (the filename without '.pack') that
2048
were found in obsolete_packs.
2051
1994
obsolete_pack_transport = self.transport.clone('obsolete_packs')
2052
if preserve is None:
2054
1995
for filename in obsolete_pack_transport.list_dir('.'):
2055
name, ext = osutils.splitext(filename)
2058
if name in preserve:
2061
1997
obsolete_pack_transport.delete(filename)
2062
1998
except (errors.PathError, errors.TransportError), e:
2063
warning("couldn't delete obsolete pack, skipping it:\n%s"
1999
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
2067
2001
def _start_write_group(self):
2068
2002
# Do not permit preparation for writing if we're not in a 'write lock'.
2095
2029
# FIXME: just drop the transient index.
2096
2030
# forget what names there are
2097
2031
if self._new_pack is not None:
2098
operation = cleanup.OperationWithCleanups(self._new_pack.abort)
2099
operation.add_cleanup(setattr, self, '_new_pack', None)
2100
# If we aborted while in the middle of finishing the write
2101
# group, _remove_pack_indices could fail because the indexes are
2102
# already gone. But they're not there we shouldn't fail in this
2103
# case, so we pass ignore_missing=True.
2104
operation.add_cleanup(self._remove_pack_indices, self._new_pack,
2105
ignore_missing=True)
2106
operation.run_simple()
2033
self._new_pack.abort()
2035
# XXX: If we aborted while in the middle of finishing the write
2036
# group, _remove_pack_indices can fail because the indexes are
2037
# already gone. If they're not there we shouldn't fail in this
2038
# case. -- mbp 20081113
2039
self._remove_pack_indices(self._new_pack)
2040
self._new_pack = None
2107
2041
for resumed_pack in self._resumed_packs:
2108
operation = cleanup.OperationWithCleanups(resumed_pack.abort)
2109
# See comment in previous finally block.
2110
operation.add_cleanup(self._remove_pack_indices, resumed_pack,
2111
ignore_missing=True)
2112
operation.run_simple()
2043
resumed_pack.abort()
2045
# See comment in previous finally block.
2047
self._remove_pack_indices(resumed_pack)
2113
2050
del self._resumed_packs[:]
2115
2052
def _remove_resumed_pack_indices(self):
2117
2054
self._remove_pack_indices(resumed_pack)
2118
2055
del self._resumed_packs[:]
2120
def _check_new_inventories(self):
2121
"""Detect missing inventories in this write group.
2123
:returns: list of strs, summarising any problems found. If the list is
2124
empty no problems were found.
2126
# The base implementation does no checks. GCRepositoryPackCollection
2130
2057
def _commit_write_group(self):
2131
2058
all_missing = set()
2132
2059
for prefix, versioned_file in (
2141
2068
raise errors.BzrCheckError(
2142
2069
"Repository %s has missing compression parent(s) %r "
2143
2070
% (self.repo, sorted(all_missing)))
2144
problems = self._check_new_inventories()
2146
problems_summary = '\n'.join(problems)
2147
raise errors.BzrCheckError(
2148
"Cannot add revision(s) to repository: " + problems_summary)
2149
2071
self._remove_pack_indices(self._new_pack)
2150
any_new_content = False
2072
should_autopack = False
2151
2073
if self._new_pack.data_inserted():
2152
2074
# get all the data to disk and read to use
2153
2075
self._new_pack.finish()
2154
2076
self.allocate(self._new_pack)
2155
2077
self._new_pack = None
2156
any_new_content = True
2078
should_autopack = True
2158
2080
self._new_pack.abort()
2159
2081
self._new_pack = None
2281
2200
self._reconcile_fixes_text_parents = True
2282
2201
self._reconcile_backsup_inventory = False
2284
def _warn_if_deprecated(self, branch=None):
2203
def _warn_if_deprecated(self):
2285
2204
# This class isn't deprecated, but one sub-format is
2286
2205
if isinstance(self._format, RepositoryFormatKnitPack5RichRootBroken):
2287
super(KnitPackRepository, self)._warn_if_deprecated(branch)
2206
from bzrlib import repository
2207
if repository._deprecation_warning_done:
2209
repository._deprecation_warning_done = True
2210
warning("Format %s for %s is deprecated - please use"
2211
" 'bzr upgrade --1.6.1-rich-root'"
2212
% (self._format, self.bzrdir.transport.base))
2289
2214
def _abort_write_group(self):
2290
self.revisions._index._key_dependencies.clear()
2215
self.revisions._index._key_dependencies.refs.clear()
2291
2216
self._pack_collection._abort_write_group()
2218
def _find_inconsistent_revision_parents(self):
2219
"""Find revisions with incorrectly cached parents.
2221
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
2222
parents-in-revision).
2224
if not self.is_locked():
2225
raise errors.ObjectNotLocked(self)
2226
pb = ui.ui_factory.nested_progress_bar()
2229
revision_nodes = self._pack_collection.revision_index \
2230
.combined_index.iter_all_entries()
2231
index_positions = []
2232
# Get the cached index values for all revisions, and also the
2233
# location in each index of the revision text so we can perform
2235
for index, key, value, refs in revision_nodes:
2236
node = (index, key, value, refs)
2237
index_memo = self.revisions._index._node_to_position(node)
2238
if index_memo[0] != index:
2239
raise AssertionError('%r != %r' % (index_memo[0], index))
2240
index_positions.append((index_memo, key[0],
2241
tuple(parent[0] for parent in refs[0])))
2242
pb.update("Reading revision index", 0, 0)
2243
index_positions.sort()
2245
pb.update("Checking cached revision graph", 0,
2246
len(index_positions))
2247
for offset in xrange(0, len(index_positions), 1000):
2248
pb.update("Checking cached revision graph", offset)
2249
to_query = index_positions[offset:offset + batch_size]
2252
rev_ids = [item[1] for item in to_query]
2253
revs = self.get_revisions(rev_ids)
2254
for revision, item in zip(revs, to_query):
2255
index_parents = item[2]
2256
rev_parents = tuple(revision.parent_ids)
2257
if index_parents != rev_parents:
2258
result.append((revision.revision_id, index_parents,
2293
2264
def _get_source(self, to_format):
2294
2265
if to_format.network_name() == self._format.network_name():
2295
2266
return KnitPackStreamSource(self, to_format)
2307
2278
self._pack_collection._start_write_group()
2309
2280
def _commit_write_group(self):
2310
hint = self._pack_collection._commit_write_group()
2311
self.revisions._index._key_dependencies.clear()
2281
self.revisions._index._key_dependencies.refs.clear()
2282
return self._pack_collection._commit_write_group()
2314
2284
def suspend_write_group(self):
2315
2285
# XXX check self._write_group is self.get_transaction()?
2316
2286
tokens = self._pack_collection._suspend_write_group()
2317
self.revisions._index._key_dependencies.clear()
2287
self.revisions._index._key_dependencies.refs.clear()
2318
2288
self._write_group = None
2348
2318
if self._write_lock_count == 1:
2349
2319
self._transaction = transactions.WriteTransaction()
2351
if 'relock' in debug.debug_flags and self._prev_lock == 'w':
2352
note('%r was write locked again', self)
2353
self._prev_lock = 'w'
2354
2321
for repo in self._fallback_repositories:
2355
2322
# Writes don't affect fallback repos
2356
2323
repo.lock_read()
2357
2324
self._refresh_data()
2358
return RepositoryWriteLockResult(self.unlock, None)
2360
2326
def lock_read(self):
2361
2327
locked = self.is_locked()
2381
2343
raise NotImplementedError(self.dont_leave_lock_in_place)
2383
2345
@needs_write_lock
2384
def pack(self, hint=None, clean_obsolete_packs=False):
2346
def pack(self, hint=None):
2385
2347
"""Compress the data within the repository.
2387
2349
This will pack all the data to a single pack. In future it may
2388
2350
recompress deltas or do other such expensive operations.
2390
self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2352
self._pack_collection.pack(hint=hint)
2392
2354
@needs_write_lock
2393
2355
def reconcile(self, other=None, thorough=False):
2549
2510
utf8_files = [('format', self.get_format_string())]
2551
2512
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2552
repository = self.open(a_bzrdir=a_bzrdir, _found=True)
2553
self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
2513
return self.open(a_bzrdir=a_bzrdir, _found=True)
2556
2515
def open(self, a_bzrdir, _found=False, _override_transport=None):
2557
2516
"""See RepositoryFormat.open().
2639
2600
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2602
def check_conversion_target(self, target_format):
2603
if not target_format.rich_root_data:
2604
raise errors.BadConversionTarget(
2605
'Does not support rich root data.', target_format)
2606
if not getattr(target_format, 'supports_tree_reference', False):
2607
raise errors.BadConversionTarget(
2608
'Does not support nested trees', target_format)
2641
2610
def get_format_string(self):
2642
2611
"""See RepositoryFormat.get_format_string()."""
2643
2612
return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
2677
2646
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2648
def check_conversion_target(self, target_format):
2649
if not target_format.rich_root_data:
2650
raise errors.BadConversionTarget(
2651
'Does not support rich root data.', target_format)
2679
2653
def get_format_string(self):
2680
2654
"""See RepositoryFormat.get_format_string()."""
2681
2655
return ("Bazaar pack repository format 1 with rich root"
2755
2732
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2734
def check_conversion_target(self, target_format):
2735
if not target_format.rich_root_data:
2736
raise errors.BadConversionTarget(
2737
'Does not support rich root data.', target_format)
2757
2739
def get_format_string(self):
2758
2740
"""See RepositoryFormat.get_format_string()."""
2759
2741
return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n"
2801
2783
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2785
def check_conversion_target(self, target_format):
2786
if not target_format.rich_root_data:
2787
raise errors.BadConversionTarget(
2788
'Does not support rich root data.', target_format)
2803
2790
def get_format_string(self):
2804
2791
"""See RepositoryFormat.get_format_string()."""
2805
2792
return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n"
2873
2863
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2865
def check_conversion_target(self, target_format):
2866
if not target_format.rich_root_data:
2867
raise errors.BadConversionTarget(
2868
'Does not support rich root data.', target_format)
2875
2870
def get_format_string(self):
2876
2871
"""See RepositoryFormat.get_format_string()."""
2877
2872
return "Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
2915
2909
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2911
def check_conversion_target(self, target_format):
2912
if not target_format.rich_root_data:
2913
raise errors.BadConversionTarget(
2914
'Does not support rich root data.', target_format)
2915
if not getattr(target_format, 'supports_tree_reference', False):
2916
raise errors.BadConversionTarget(
2917
'Does not support nested trees', target_format)
2917
2919
def get_format_string(self):
2918
2920
"""See RepositoryFormat.get_format_string()."""
2919
2921
return ("Bazaar development format 2 with subtree support "
added
removed
bzrlib/repofmt/pack_repo.py
