1
# Copyright (C) 2007-2010 Canonical Ltd
1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
55
54
revision as _mod_revision,
58
from bzrlib.decorators import needs_write_lock, only_raises
57
from bzrlib.decorators import needs_write_lock
59
58
from bzrlib.btree_index import (
227
224
return self.index_name('text', name)
229
226
def _replace_index_with_readonly(self, index_type):
230
unlimited_cache = False
231
if index_type == 'chk':
232
unlimited_cache = True
233
227
setattr(self, index_type + '_index',
234
228
self.index_class(self.index_transport,
235
229
self.index_name(index_type, self.name),
236
self.index_sizes[self.index_offset(index_type)],
237
unlimited_cache=unlimited_cache))
230
self.index_sizes[self.index_offset(index_type)]))
240
233
class ExistingPack(Pack):
588
581
flush_func=flush_func)
589
582
self.add_callback = None
584
def replace_indices(self, index_to_pack, indices):
585
"""Replace the current mappings with fresh ones.
587
This should probably not be used eventually, rather incremental add and
588
removal of indices. It has been added during refactoring of existing
591
:param index_to_pack: A mapping from index objects to
592
(transport, name) tuples for the pack file data.
593
:param indices: A list of indices.
595
# refresh the revision pack map dict without replacing the instance.
596
self.index_to_pack.clear()
597
self.index_to_pack.update(index_to_pack)
598
# XXX: API break - clearly a 'replace' method would be good?
599
self.combined_index._indices[:] = indices
600
# the current add nodes callback for the current writable index if
602
self.add_callback = None
591
604
def add_index(self, index, pack):
592
605
"""Add index to the aggregate, which is an index for Pack pack.
600
613
# expose it to the index map
601
614
self.index_to_pack[index] = pack.access_tuple()
602
615
# put it at the front of the linear index list
603
self.combined_index.insert_index(0, index, pack.name)
616
self.combined_index.insert_index(0, index)
605
618
def add_writable_index(self, index, pack):
606
619
"""Add an index which is able to have data added to it.
626
639
self.data_access.set_writer(None, None, (None, None))
627
640
self.index_to_pack.clear()
628
641
del self.combined_index._indices[:]
629
del self.combined_index._index_names[:]
630
642
self.add_callback = None
632
def remove_index(self, index):
644
def remove_index(self, index, pack):
633
645
"""Remove index from the indices used to answer queries.
635
647
:param index: An index from the pack parameter.
648
:param pack: A Pack instance.
637
650
del self.index_to_pack[index]
638
pos = self.combined_index._indices.index(index)
639
del self.combined_index._indices[pos]
640
del self.combined_index._index_names[pos]
651
self.combined_index._indices.remove(index)
641
652
if (self.add_callback is not None and
642
653
getattr(index, 'add_nodes', None) == self.add_callback):
643
654
self.add_callback = None
1101
1112
iterator is a tuple with:
1102
1113
index, readv_vector, node_vector. readv_vector is a list ready to
1103
1114
hand to the transport readv method, and node_vector is a list of
1104
(key, eol_flag, references) for the node retrieved by the
1115
(key, eol_flag, references) for the the node retrieved by the
1105
1116
matching readv_vector.
1107
1118
# group by pack so we do one readv per pack
1399
1410
self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1400
1411
self.text_index = AggregateIndex(self.reload_pack_names, flush)
1401
1412
self.signature_index = AggregateIndex(self.reload_pack_names, flush)
1402
all_indices = [self.revision_index, self.inventory_index,
1403
self.text_index, self.signature_index]
1404
1413
if use_chk_index:
1405
1414
self.chk_index = AggregateIndex(self.reload_pack_names, flush)
1406
all_indices.append(self.chk_index)
1408
1416
# used to determine if we're using a chk_index elsewhere.
1409
1417
self.chk_index = None
1410
# Tell all the CombinedGraphIndex objects about each other, so they can
1411
# share hints about which pack names to search first.
1412
all_combined = [agg_idx.combined_index for agg_idx in all_indices]
1413
for combined_idx in all_combined:
1414
combined_idx.set_sibling_indices(
1415
set(all_combined).difference([combined_idx]))
1416
1418
# resumed packs
1417
1419
self._resumed_packs = []
1420
return '%s(%r)' % (self.__class__.__name__, self.repo)
1422
1421
def add_pack_to_memory(self, pack):
1423
1422
"""Make a Pack object available to the repository to satisfy queries.
1538
1537
self._remove_pack_from_memory(pack)
1539
1538
# record the newly available packs and stop advertising the old
1541
to_be_obsoleted = []
1542
for _, packs in pack_operations:
1543
to_be_obsoleted.extend(packs)
1544
result = self._save_pack_names(clear_obsolete_packs=True,
1545
obsolete_packs=to_be_obsoleted)
1540
result = self._save_pack_names(clear_obsolete_packs=True)
1541
# Move the old packs out of the way now they are no longer referenced.
1542
for revision_count, packs in pack_operations:
1543
self._obsolete_packs(packs)
1548
1546
def _flush_new_pack(self):
1561
1559
"""Is the collection already packed?"""
1562
1560
return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1564
def pack(self, hint=None, clean_obsolete_packs=False):
1562
def pack(self, hint=None):
1565
1563
"""Pack the pack collection totally."""
1566
1564
self.ensure_loaded()
1567
1565
total_packs = len(self._names)
1583
1581
pack_operations[-1][1].append(pack)
1584
1582
self._execute_pack_operations(pack_operations, OptimisingPacker)
1586
if clean_obsolete_packs:
1587
self._clear_obsolete_packs()
1589
1584
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1590
1585
"""Plan a pack operation.
1679
1674
txt_index = self._make_index(name, '.tix')
1680
1675
sig_index = self._make_index(name, '.six')
1681
1676
if self.chk_index is not None:
1682
chk_index = self._make_index(name, '.cix', unlimited_cache=True)
1677
chk_index = self._make_index(name, '.cix')
1684
1679
chk_index = None
1685
1680
result = ExistingPack(self._pack_transport, name, rev_index,
1704
1699
txt_index = self._make_index(name, '.tix', resume=True)
1705
1700
sig_index = self._make_index(name, '.six', resume=True)
1706
1701
if self.chk_index is not None:
1707
chk_index = self._make_index(name, '.cix', resume=True,
1708
unlimited_cache=True)
1702
chk_index = self._make_index(name, '.cix', resume=True)
1710
1704
chk_index = None
1711
1705
result = self.resumed_pack_factory(name, rev_index, inv_index,
1741
1735
return self._index_class(self.transport, 'pack-names', None
1742
1736
).iter_all_entries()
1744
def _make_index(self, name, suffix, resume=False, unlimited_cache=False):
1738
def _make_index(self, name, suffix, resume=False):
1745
1739
size_offset = self._suffix_offsets[suffix]
1746
1740
index_name = name + suffix
1751
1745
transport = self._index_transport
1752
1746
index_size = self._names[name][size_offset]
1753
return self._index_class(transport, index_name, index_size,
1754
unlimited_cache=unlimited_cache)
1747
return self._index_class(transport, index_name, index_size)
1756
1749
def _max_pack_count(self, total_revisions):
1757
1750
"""Return the maximum number of packs to use for total revisions.
1785
1778
:param return: None.
1787
1780
for pack in packs:
1789
pack.pack_transport.rename(pack.file_name(),
1790
'../obsolete_packs/' + pack.file_name())
1791
except (errors.PathError, errors.TransportError), e:
1792
# TODO: Should these be warnings or mutters?
1793
mutter("couldn't rename obsolete pack, skipping it:\n%s"
1781
pack.pack_transport.rename(pack.file_name(),
1782
'../obsolete_packs/' + pack.file_name())
1795
1783
# TODO: Probably needs to know all possible indices for this pack
1796
1784
# - or maybe list the directory and move all indices matching this
1797
1785
# name whether we recognize it or not?
1799
1787
if self.chk_index is not None:
1800
1788
suffixes.append('.cix')
1801
1789
for suffix in suffixes:
1803
self._index_transport.rename(pack.name + suffix,
1804
'../obsolete_packs/' + pack.name + suffix)
1805
except (errors.PathError, errors.TransportError), e:
1806
mutter("couldn't rename obsolete index, skipping it:\n%s"
1790
self._index_transport.rename(pack.name + suffix,
1791
'../obsolete_packs/' + pack.name + suffix)
1809
1793
def pack_distribution(self, total_revisions):
1810
1794
"""Generate a list of the number of revisions to put in each pack.
1836
1820
self._remove_pack_indices(pack)
1837
1821
self.packs.remove(pack)
1839
def _remove_pack_indices(self, pack, ignore_missing=False):
1840
"""Remove the indices for pack from the aggregated indices.
1842
:param ignore_missing: Suppress KeyErrors from calling remove_index.
1844
for index_type in Pack.index_definitions.keys():
1845
attr_name = index_type + '_index'
1846
aggregate_index = getattr(self, attr_name)
1847
if aggregate_index is not None:
1848
pack_index = getattr(pack, attr_name)
1850
aggregate_index.remove_index(pack_index)
1823
def _remove_pack_indices(self, pack):
1824
"""Remove the indices for pack from the aggregated indices."""
1825
self.revision_index.remove_index(pack.revision_index, pack)
1826
self.inventory_index.remove_index(pack.inventory_index, pack)
1827
self.text_index.remove_index(pack.text_index, pack)
1828
self.signature_index.remove_index(pack.signature_index, pack)
1829
if self.chk_index is not None:
1830
self.chk_index.remove_index(pack.chk_index, pack)
1856
1832
def reset(self):
1857
1833
"""Clear all cached data."""
1890
1866
disk_nodes = set()
1891
1867
for index, key, value in self._iter_disk_pack_index():
1892
1868
disk_nodes.add((key, value))
1893
orig_disk_nodes = set(disk_nodes)
1895
1870
# do a two-way diff against our original content
1896
1871
current_nodes = set()
1909
1884
disk_nodes.difference_update(deleted_nodes)
1910
1885
disk_nodes.update(new_nodes)
1912
return disk_nodes, deleted_nodes, new_nodes, orig_disk_nodes
1887
return disk_nodes, deleted_nodes, new_nodes
1914
1889
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1915
1890
"""Given the correct set of pack files, update our saved info.
1955
1930
added.append(name)
1956
1931
return removed, added, modified
1958
def _save_pack_names(self, clear_obsolete_packs=False, obsolete_packs=None):
1933
def _save_pack_names(self, clear_obsolete_packs=False):
1959
1934
"""Save the list of packs.
1961
1936
This will take out the mutex around the pack names list for the
1966
1941
:param clear_obsolete_packs: If True, clear out the contents of the
1967
1942
obsolete_packs directory.
1968
:param obsolete_packs: Packs that are obsolete once the new pack-names
1969
file has been written.
1970
1943
:return: A list of the names saved that were not previously on disk.
1972
already_obsolete = []
1973
1945
self.lock_names()
1975
1947
builder = self._index_builder_class()
1976
(disk_nodes, deleted_nodes, new_nodes,
1977
orig_disk_nodes) = self._diff_pack_names()
1948
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1978
1949
# TODO: handle same-name, index-size-changes here -
1979
1950
# e.g. use the value from disk, not ours, *unless* we're the one
1982
1953
builder.add_node(key, value)
1983
1954
self.transport.put_file('pack-names', builder.finish(),
1984
1955
mode=self.repo.bzrdir._get_file_mode())
1956
# move the baseline forward
1985
1957
self._packs_at_load = disk_nodes
1986
1958
if clear_obsolete_packs:
1989
to_preserve = set([o.name for o in obsolete_packs])
1990
already_obsolete = self._clear_obsolete_packs(to_preserve)
1959
self._clear_obsolete_packs()
1992
1961
self._unlock_names()
1993
1962
# synchronise the memory packs list with what we just wrote:
1994
1963
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1996
# TODO: We could add one more condition here. "if o.name not in
1997
# orig_disk_nodes and o != the new_pack we haven't written to
1998
# disk yet. However, the new pack object is not easily
1999
# accessible here (it would have to be passed through the
2000
# autopacking code, etc.)
2001
obsolete_packs = [o for o in obsolete_packs
2002
if o.name not in already_obsolete]
2003
self._obsolete_packs(obsolete_packs)
2004
1964
return [new_node[0][0] for new_node in new_nodes]
2006
1966
def reload_pack_names(self):
2023
1983
# out the new value.
2024
(disk_nodes, deleted_nodes, new_nodes,
2025
orig_disk_nodes) = self._diff_pack_names()
2026
# _packs_at_load is meant to be the explicit list of names in
2027
# 'pack-names' at then start. As such, it should not contain any
2028
# pending names that haven't been written out yet.
2029
self._packs_at_load = orig_disk_nodes
1984
disk_nodes, _, _ = self._diff_pack_names()
1985
self._packs_at_load = disk_nodes
2030
1986
(removed, added,
2031
1987
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
2032
1988
if removed or added or modified:
2042
1998
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
2044
def _clear_obsolete_packs(self, preserve=None):
2000
def _clear_obsolete_packs(self):
2045
2001
"""Delete everything from the obsolete-packs directory.
2047
:return: A list of pack identifiers (the filename without '.pack') that
2048
were found in obsolete_packs.
2051
2003
obsolete_pack_transport = self.transport.clone('obsolete_packs')
2052
if preserve is None:
2054
2004
for filename in obsolete_pack_transport.list_dir('.'):
2055
name, ext = osutils.splitext(filename)
2058
if name in preserve:
2061
2006
obsolete_pack_transport.delete(filename)
2062
2007
except (errors.PathError, errors.TransportError), e:
2063
warning("couldn't delete obsolete pack, skipping it:\n%s"
2008
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
2067
2010
def _start_write_group(self):
2068
2011
# Do not permit preparation for writing if we're not in a 'write lock'.
2095
2038
# FIXME: just drop the transient index.
2096
2039
# forget what names there are
2097
2040
if self._new_pack is not None:
2098
operation = cleanup.OperationWithCleanups(self._new_pack.abort)
2099
operation.add_cleanup(setattr, self, '_new_pack', None)
2100
# If we aborted while in the middle of finishing the write
2101
# group, _remove_pack_indices could fail because the indexes are
2102
# already gone. But they're not there we shouldn't fail in this
2103
# case, so we pass ignore_missing=True.
2104
operation.add_cleanup(self._remove_pack_indices, self._new_pack,
2105
ignore_missing=True)
2106
operation.run_simple()
2042
self._new_pack.abort()
2044
# XXX: If we aborted while in the middle of finishing the write
2045
# group, _remove_pack_indices can fail because the indexes are
2046
# already gone. If they're not there we shouldn't fail in this
2047
# case. -- mbp 20081113
2048
self._remove_pack_indices(self._new_pack)
2049
self._new_pack = None
2107
2050
for resumed_pack in self._resumed_packs:
2108
operation = cleanup.OperationWithCleanups(resumed_pack.abort)
2109
# See comment in previous finally block.
2110
operation.add_cleanup(self._remove_pack_indices, resumed_pack,
2111
ignore_missing=True)
2112
operation.run_simple()
2052
resumed_pack.abort()
2054
# See comment in previous finally block.
2056
self._remove_pack_indices(resumed_pack)
2113
2059
del self._resumed_packs[:]
2115
2061
def _remove_resumed_pack_indices(self):
2117
2063
self._remove_pack_indices(resumed_pack)
2118
2064
del self._resumed_packs[:]
2120
def _check_new_inventories(self):
2121
"""Detect missing inventories in this write group.
2123
:returns: list of strs, summarising any problems found. If the list is
2124
empty no problems were found.
2126
# The base implementation does no checks. GCRepositoryPackCollection
2130
2066
def _commit_write_group(self):
2131
2067
all_missing = set()
2132
2068
for prefix, versioned_file in (
2141
2077
raise errors.BzrCheckError(
2142
2078
"Repository %s has missing compression parent(s) %r "
2143
2079
% (self.repo, sorted(all_missing)))
2144
problems = self._check_new_inventories()
2146
problems_summary = '\n'.join(problems)
2147
raise errors.BzrCheckError(
2148
"Cannot add revision(s) to repository: " + problems_summary)
2149
2080
self._remove_pack_indices(self._new_pack)
2150
any_new_content = False
2081
should_autopack = False
2151
2082
if self._new_pack.data_inserted():
2152
2083
# get all the data to disk and read to use
2153
2084
self._new_pack.finish()
2154
2085
self.allocate(self._new_pack)
2155
2086
self._new_pack = None
2156
any_new_content = True
2087
should_autopack = True
2158
2089
self._new_pack.abort()
2159
2090
self._new_pack = None
2164
2095
self._remove_pack_from_memory(resumed_pack)
2165
2096
resumed_pack.finish()
2166
2097
self.allocate(resumed_pack)
2167
any_new_content = True
2098
should_autopack = True
2168
2099
del self._resumed_packs[:]
2170
result = self.autopack()
2101
if not self.autopack():
2172
2102
# when autopack takes no steps, the names list is still
2174
2104
return self._save_pack_names()
2178
2107
def _suspend_write_group(self):
2281
2210
self._reconcile_fixes_text_parents = True
2282
2211
self._reconcile_backsup_inventory = False
2284
def _warn_if_deprecated(self, branch=None):
2213
def _warn_if_deprecated(self):
2285
2214
# This class isn't deprecated, but one sub-format is
2286
2215
if isinstance(self._format, RepositoryFormatKnitPack5RichRootBroken):
2287
super(KnitPackRepository, self)._warn_if_deprecated(branch)
2216
from bzrlib import repository
2217
if repository._deprecation_warning_done:
2219
repository._deprecation_warning_done = True
2220
warning("Format %s for %s is deprecated - please use"
2221
" 'bzr upgrade --1.6.1-rich-root'"
2222
% (self._format, self.bzrdir.transport.base))
2289
2224
def _abort_write_group(self):
2290
self.revisions._index._key_dependencies.clear()
2225
self.revisions._index._key_dependencies.refs.clear()
2291
2226
self._pack_collection._abort_write_group()
2293
2228
def _get_source(self, to_format):
2307
2242
self._pack_collection._start_write_group()
2309
2244
def _commit_write_group(self):
2310
hint = self._pack_collection._commit_write_group()
2311
self.revisions._index._key_dependencies.clear()
2245
self.revisions._index._key_dependencies.refs.clear()
2246
return self._pack_collection._commit_write_group()
2314
2248
def suspend_write_group(self):
2315
2249
# XXX check self._write_group is self.get_transaction()?
2316
2250
tokens = self._pack_collection._suspend_write_group()
2317
self.revisions._index._key_dependencies.clear()
2251
self.revisions._index._key_dependencies.refs.clear()
2318
2252
self._write_group = None
2348
2282
if self._write_lock_count == 1:
2349
2283
self._transaction = transactions.WriteTransaction()
2351
if 'relock' in debug.debug_flags and self._prev_lock == 'w':
2352
note('%r was write locked again', self)
2353
self._prev_lock = 'w'
2354
2285
for repo in self._fallback_repositories:
2355
2286
# Writes don't affect fallback repos
2356
2287
repo.lock_read()
2357
2288
self._refresh_data()
2358
return RepositoryWriteLockResult(self.unlock, None)
2360
2290
def lock_read(self):
2361
2291
locked = self.is_locked()
2365
2295
self.control_files.lock_read()
2367
if 'relock' in debug.debug_flags and self._prev_lock == 'r':
2368
note('%r was read locked again', self)
2369
self._prev_lock = 'r'
2370
2297
for repo in self._fallback_repositories:
2371
2298
repo.lock_read()
2372
2299
self._refresh_data()
2375
2301
def leave_lock_in_place(self):
2376
2302
# not supported - raise an error
2381
2307
raise NotImplementedError(self.dont_leave_lock_in_place)
2383
2309
@needs_write_lock
2384
def pack(self, hint=None, clean_obsolete_packs=False):
2310
def pack(self, hint=None):
2385
2311
"""Compress the data within the repository.
2387
2313
This will pack all the data to a single pack. In future it may
2388
2314
recompress deltas or do other such expensive operations.
2390
self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2316
self._pack_collection.pack(hint=hint)
2392
2318
@needs_write_lock
2393
2319
def reconcile(self, other=None, thorough=False):
2401
2327
packer = ReconcilePacker(collection, packs, extension, revs)
2402
2328
return packer.pack(pb)
2404
@only_raises(errors.LockNotHeld, errors.LockBroken)
2405
2330
def unlock(self):
2406
2331
if self._write_lock_count == 1 and self._write_group is not None:
2407
2332
self.abort_write_group()
2549
2474
utf8_files = [('format', self.get_format_string())]
2551
2476
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2552
repository = self.open(a_bzrdir=a_bzrdir, _found=True)
2553
self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
2477
return self.open(a_bzrdir=a_bzrdir, _found=True)
2556
2479
def open(self, a_bzrdir, _found=False, _override_transport=None):
2557
2480
"""See RepositoryFormat.open().
2894
2816
repository_class = KnitPackRepository
2895
2817
_commit_builder_class = PackRootCommitBuilder
2896
2818
rich_root_data = True
2898
2819
supports_tree_reference = True
2899
2820
supports_external_lookups = True
2900
2821
# What index classes to use
added
removed
bzrlib/repofmt/pack_repo.py
