1
# Copyright (C) 2007-2010 Canonical Ltd
1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
588
586
flush_func=flush_func)
589
587
self.add_callback = None
589
def replace_indices(self, index_to_pack, indices):
590
"""Replace the current mappings with fresh ones.
592
This should probably not be used eventually, rather incremental add and
593
removal of indices. It has been added during refactoring of existing
596
:param index_to_pack: A mapping from index objects to
597
(transport, name) tuples for the pack file data.
598
:param indices: A list of indices.
600
# refresh the revision pack map dict without replacing the instance.
601
self.index_to_pack.clear()
602
self.index_to_pack.update(index_to_pack)
603
# XXX: API break - clearly a 'replace' method would be good?
604
self.combined_index._indices[:] = indices
605
# the current add nodes callback for the current writable index if
607
self.add_callback = None
591
609
def add_index(self, index, pack):
592
610
"""Add index to the aggregate, which is an index for Pack pack.
600
618
# expose it to the index map
601
619
self.index_to_pack[index] = pack.access_tuple()
602
620
# put it at the front of the linear index list
603
self.combined_index.insert_index(0, index, pack.name)
621
self.combined_index.insert_index(0, index)
605
623
def add_writable_index(self, index, pack):
606
624
"""Add an index which is able to have data added to it.
626
644
self.data_access.set_writer(None, None, (None, None))
627
645
self.index_to_pack.clear()
628
646
del self.combined_index._indices[:]
629
del self.combined_index._index_names[:]
630
647
self.add_callback = None
632
def remove_index(self, index):
649
def remove_index(self, index, pack):
633
650
"""Remove index from the indices used to answer queries.
635
652
:param index: An index from the pack parameter.
653
:param pack: A Pack instance.
637
655
del self.index_to_pack[index]
638
pos = self.combined_index._indices.index(index)
639
del self.combined_index._indices[pos]
640
del self.combined_index._index_names[pos]
656
self.combined_index._indices.remove(index)
641
657
if (self.add_callback is not None and
642
658
getattr(index, 'add_nodes', None) == self.add_callback):
643
659
self.add_callback = None
1399
1415
self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1400
1416
self.text_index = AggregateIndex(self.reload_pack_names, flush)
1401
1417
self.signature_index = AggregateIndex(self.reload_pack_names, flush)
1402
all_indices = [self.revision_index, self.inventory_index,
1403
self.text_index, self.signature_index]
1404
1418
if use_chk_index:
1405
1419
self.chk_index = AggregateIndex(self.reload_pack_names, flush)
1406
all_indices.append(self.chk_index)
1408
1421
# used to determine if we're using a chk_index elsewhere.
1409
1422
self.chk_index = None
1410
# Tell all the CombinedGraphIndex objects about each other, so they can
1411
# share hints about which pack names to search first.
1412
all_combined = [agg_idx.combined_index for agg_idx in all_indices]
1413
for combined_idx in all_combined:
1414
combined_idx.set_sibling_indices(
1415
set(all_combined).difference([combined_idx]))
1416
1423
# resumed packs
1417
1424
self._resumed_packs = []
1538
1545
self._remove_pack_from_memory(pack)
1539
1546
# record the newly available packs and stop advertising the old
1541
to_be_obsoleted = []
1542
for _, packs in pack_operations:
1543
to_be_obsoleted.extend(packs)
1544
result = self._save_pack_names(clear_obsolete_packs=True,
1545
obsolete_packs=to_be_obsoleted)
1548
result = self._save_pack_names(clear_obsolete_packs=True)
1549
# Move the old packs out of the way now they are no longer referenced.
1550
for revision_count, packs in pack_operations:
1551
self._obsolete_packs(packs)
1548
1554
def _flush_new_pack(self):
1561
1567
"""Is the collection already packed?"""
1562
1568
return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1564
def pack(self, hint=None, clean_obsolete_packs=False):
1570
def pack(self, hint=None):
1565
1571
"""Pack the pack collection totally."""
1566
1572
self.ensure_loaded()
1567
1573
total_packs = len(self._names)
1583
1589
pack_operations[-1][1].append(pack)
1584
1590
self._execute_pack_operations(pack_operations, OptimisingPacker)
1586
if clean_obsolete_packs:
1587
self._clear_obsolete_packs()
1589
1592
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1590
1593
"""Plan a pack operation.
1785
1788
:param return: None.
1787
1790
for pack in packs:
1789
pack.pack_transport.rename(pack.file_name(),
1790
'../obsolete_packs/' + pack.file_name())
1791
except (errors.PathError, errors.TransportError), e:
1792
# TODO: Should these be warnings or mutters?
1793
mutter("couldn't rename obsolete pack, skipping it:\n%s"
1791
pack.pack_transport.rename(pack.file_name(),
1792
'../obsolete_packs/' + pack.file_name())
1795
1793
# TODO: Probably needs to know all possible indices for this pack
1796
1794
# - or maybe list the directory and move all indices matching this
1797
1795
# name whether we recognize it or not?
1799
1797
if self.chk_index is not None:
1800
1798
suffixes.append('.cix')
1801
1799
for suffix in suffixes:
1803
self._index_transport.rename(pack.name + suffix,
1804
'../obsolete_packs/' + pack.name + suffix)
1805
except (errors.PathError, errors.TransportError), e:
1806
mutter("couldn't rename obsolete index, skipping it:\n%s"
1800
self._index_transport.rename(pack.name + suffix,
1801
'../obsolete_packs/' + pack.name + suffix)
1809
1803
def pack_distribution(self, total_revisions):
1810
1804
"""Generate a list of the number of revisions to put in each pack.
1836
1830
self._remove_pack_indices(pack)
1837
1831
self.packs.remove(pack)
1839
def _remove_pack_indices(self, pack, ignore_missing=False):
1840
"""Remove the indices for pack from the aggregated indices.
1842
:param ignore_missing: Suppress KeyErrors from calling remove_index.
1844
for index_type in Pack.index_definitions.keys():
1845
attr_name = index_type + '_index'
1846
aggregate_index = getattr(self, attr_name)
1847
if aggregate_index is not None:
1848
pack_index = getattr(pack, attr_name)
1850
aggregate_index.remove_index(pack_index)
1833
def _remove_pack_indices(self, pack):
1834
"""Remove the indices for pack from the aggregated indices."""
1835
self.revision_index.remove_index(pack.revision_index, pack)
1836
self.inventory_index.remove_index(pack.inventory_index, pack)
1837
self.text_index.remove_index(pack.text_index, pack)
1838
self.signature_index.remove_index(pack.signature_index, pack)
1839
if self.chk_index is not None:
1840
self.chk_index.remove_index(pack.chk_index, pack)
1856
1842
def reset(self):
1857
1843
"""Clear all cached data."""
1890
1876
disk_nodes = set()
1891
1877
for index, key, value in self._iter_disk_pack_index():
1892
1878
disk_nodes.add((key, value))
1893
orig_disk_nodes = set(disk_nodes)
1895
1880
# do a two-way diff against our original content
1896
1881
current_nodes = set()
1909
1894
disk_nodes.difference_update(deleted_nodes)
1910
1895
disk_nodes.update(new_nodes)
1912
return disk_nodes, deleted_nodes, new_nodes, orig_disk_nodes
1897
return disk_nodes, deleted_nodes, new_nodes
1914
1899
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1915
1900
"""Given the correct set of pack files, update our saved info.
1955
1940
added.append(name)
1956
1941
return removed, added, modified
1958
def _save_pack_names(self, clear_obsolete_packs=False, obsolete_packs=None):
1943
def _save_pack_names(self, clear_obsolete_packs=False):
1959
1944
"""Save the list of packs.
1961
1946
This will take out the mutex around the pack names list for the
1966
1951
:param clear_obsolete_packs: If True, clear out the contents of the
1967
1952
obsolete_packs directory.
1968
:param obsolete_packs: Packs that are obsolete once the new pack-names
1969
file has been written.
1970
1953
:return: A list of the names saved that were not previously on disk.
1972
already_obsolete = []
1973
1955
self.lock_names()
1975
1957
builder = self._index_builder_class()
1976
(disk_nodes, deleted_nodes, new_nodes,
1977
orig_disk_nodes) = self._diff_pack_names()
1958
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1978
1959
# TODO: handle same-name, index-size-changes here -
1979
1960
# e.g. use the value from disk, not ours, *unless* we're the one
1982
1963
builder.add_node(key, value)
1983
1964
self.transport.put_file('pack-names', builder.finish(),
1984
1965
mode=self.repo.bzrdir._get_file_mode())
1966
# move the baseline forward
1985
1967
self._packs_at_load = disk_nodes
1986
1968
if clear_obsolete_packs:
1989
to_preserve = set([o.name for o in obsolete_packs])
1990
already_obsolete = self._clear_obsolete_packs(to_preserve)
1969
self._clear_obsolete_packs()
1992
1971
self._unlock_names()
1993
1972
# synchronise the memory packs list with what we just wrote:
1994
1973
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1996
# TODO: We could add one more condition here. "if o.name not in
1997
# orig_disk_nodes and o != the new_pack we haven't written to
1998
# disk yet. However, the new pack object is not easily
1999
# accessible here (it would have to be passed through the
2000
# autopacking code, etc.)
2001
obsolete_packs = [o for o in obsolete_packs
2002
if o.name not in already_obsolete]
2003
self._obsolete_packs(obsolete_packs)
2004
1974
return [new_node[0][0] for new_node in new_nodes]
2006
1976
def reload_pack_names(self):
2023
1993
# out the new value.
2024
(disk_nodes, deleted_nodes, new_nodes,
2025
orig_disk_nodes) = self._diff_pack_names()
2026
# _packs_at_load is meant to be the explicit list of names in
2027
# 'pack-names' at then start. As such, it should not contain any
2028
# pending names that haven't been written out yet.
2029
self._packs_at_load = orig_disk_nodes
1994
disk_nodes, _, _ = self._diff_pack_names()
1995
self._packs_at_load = disk_nodes
2030
1996
(removed, added,
2031
1997
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
2032
1998
if removed or added or modified:
2042
2008
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
2044
def _clear_obsolete_packs(self, preserve=None):
2010
def _clear_obsolete_packs(self):
2045
2011
"""Delete everything from the obsolete-packs directory.
2047
:return: A list of pack identifiers (the filename without '.pack') that
2048
were found in obsolete_packs.
2051
2013
obsolete_pack_transport = self.transport.clone('obsolete_packs')
2052
if preserve is None:
2054
2014
for filename in obsolete_pack_transport.list_dir('.'):
2055
name, ext = osutils.splitext(filename)
2058
if name in preserve:
2061
2016
obsolete_pack_transport.delete(filename)
2062
2017
except (errors.PathError, errors.TransportError), e:
2063
warning("couldn't delete obsolete pack, skipping it:\n%s"
2018
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
2067
2020
def _start_write_group(self):
2068
2021
# Do not permit preparation for writing if we're not in a 'write lock'.
2095
2048
# FIXME: just drop the transient index.
2096
2049
# forget what names there are
2097
2050
if self._new_pack is not None:
2098
operation = cleanup.OperationWithCleanups(self._new_pack.abort)
2099
operation.add_cleanup(setattr, self, '_new_pack', None)
2100
# If we aborted while in the middle of finishing the write
2101
# group, _remove_pack_indices could fail because the indexes are
2102
# already gone. But they're not there we shouldn't fail in this
2103
# case, so we pass ignore_missing=True.
2104
operation.add_cleanup(self._remove_pack_indices, self._new_pack,
2105
ignore_missing=True)
2106
operation.run_simple()
2052
self._new_pack.abort()
2054
# XXX: If we aborted while in the middle of finishing the write
2055
# group, _remove_pack_indices can fail because the indexes are
2056
# already gone. If they're not there we shouldn't fail in this
2057
# case. -- mbp 20081113
2058
self._remove_pack_indices(self._new_pack)
2059
self._new_pack = None
2107
2060
for resumed_pack in self._resumed_packs:
2108
operation = cleanup.OperationWithCleanups(resumed_pack.abort)
2109
# See comment in previous finally block.
2110
operation.add_cleanup(self._remove_pack_indices, resumed_pack,
2111
ignore_missing=True)
2112
operation.run_simple()
2062
resumed_pack.abort()
2064
# See comment in previous finally block.
2066
self._remove_pack_indices(resumed_pack)
2113
2069
del self._resumed_packs[:]
2115
2071
def _remove_resumed_pack_indices(self):
2381
2335
raise NotImplementedError(self.dont_leave_lock_in_place)
2383
2337
@needs_write_lock
2384
def pack(self, hint=None, clean_obsolete_packs=False):
2338
def pack(self, hint=None):
2385
2339
"""Compress the data within the repository.
2387
2341
This will pack all the data to a single pack. In future it may
2388
2342
recompress deltas or do other such expensive operations.
2390
self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2344
self._pack_collection.pack(hint=hint)
2392
2346
@needs_write_lock
2393
2347
def reconcile(self, other=None, thorough=False):
2549
2503
utf8_files = [('format', self.get_format_string())]
2551
2505
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2552
repository = self.open(a_bzrdir=a_bzrdir, _found=True)
2553
self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
2506
return self.open(a_bzrdir=a_bzrdir, _found=True)
2556
2508
def open(self, a_bzrdir, _found=False, _override_transport=None):
2557
2509
"""See RepositoryFormat.open().
2894
2845
repository_class = KnitPackRepository
2895
2846
_commit_builder_class = PackRootCommitBuilder
2896
2847
rich_root_data = True
2898
2848
supports_tree_reference = True
2899
2849
supports_external_lookups = True
2900
2850
# What index classes to use
added
removed
bzrlib/repofmt/pack_repo.py
