587
586
flush_func=flush_func)
588
587
self.add_callback = None
589
def replace_indices(self, index_to_pack, indices):
590
"""Replace the current mappings with fresh ones.
592
This should probably not be used eventually, rather incremental add and
593
removal of indices. It has been added during refactoring of existing
596
:param index_to_pack: A mapping from index objects to
597
(transport, name) tuples for the pack file data.
598
:param indices: A list of indices.
600
# refresh the revision pack map dict without replacing the instance.
601
self.index_to_pack.clear()
602
self.index_to_pack.update(index_to_pack)
603
# XXX: API break - clearly a 'replace' method would be good?
604
self.combined_index._indices[:] = indices
605
# the current add nodes callback for the current writable index if
607
self.add_callback = None
590
609
def add_index(self, index, pack):
591
610
"""Add index to the aggregate, which is an index for Pack pack.
599
618
# expose it to the index map
600
619
self.index_to_pack[index] = pack.access_tuple()
601
620
# put it at the front of the linear index list
602
self.combined_index.insert_index(0, index, pack.name)
621
self.combined_index.insert_index(0, index)
604
623
def add_writable_index(self, index, pack):
605
624
"""Add an index which is able to have data added to it.
625
644
self.data_access.set_writer(None, None, (None, None))
626
645
self.index_to_pack.clear()
627
646
del self.combined_index._indices[:]
628
del self.combined_index._index_names[:]
629
647
self.add_callback = None
631
def remove_index(self, index):
649
def remove_index(self, index, pack):
632
650
"""Remove index from the indices used to answer queries.
634
652
:param index: An index from the pack parameter.
653
:param pack: A Pack instance.
636
655
del self.index_to_pack[index]
637
pos = self.combined_index._indices.index(index)
638
del self.combined_index._indices[pos]
639
del self.combined_index._index_names[pos]
656
self.combined_index._indices.remove(index)
640
657
if (self.add_callback is not None and
641
658
getattr(index, 'add_nodes', None) == self.add_callback):
642
659
self.add_callback = None
1398
1415
self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1399
1416
self.text_index = AggregateIndex(self.reload_pack_names, flush)
1400
1417
self.signature_index = AggregateIndex(self.reload_pack_names, flush)
1401
all_indices = [self.revision_index, self.inventory_index,
1402
self.text_index, self.signature_index]
1403
1418
if use_chk_index:
1404
1419
self.chk_index = AggregateIndex(self.reload_pack_names, flush)
1405
all_indices.append(self.chk_index)
1407
1421
# used to determine if we're using a chk_index elsewhere.
1408
1422
self.chk_index = None
1409
# Tell all the CombinedGraphIndex objects about each other, so they can
1410
# share hints about which pack names to search first.
1411
all_combined = [agg_idx.combined_index for agg_idx in all_indices]
1412
for combined_idx in all_combined:
1413
combined_idx.set_sibling_indices(
1414
set(all_combined).difference([combined_idx]))
1415
1423
# resumed packs
1416
1424
self._resumed_packs = []
1537
1545
self._remove_pack_from_memory(pack)
1538
1546
# record the newly available packs and stop advertising the old
1540
to_be_obsoleted = []
1541
for _, packs in pack_operations:
1542
to_be_obsoleted.extend(packs)
1543
result = self._save_pack_names(clear_obsolete_packs=True,
1544
obsolete_packs=to_be_obsoleted)
1548
result = self._save_pack_names(clear_obsolete_packs=True)
1549
# Move the old packs out of the way now they are no longer referenced.
1550
for revision_count, packs in pack_operations:
1551
self._obsolete_packs(packs)
1547
1554
def _flush_new_pack(self):
1560
1567
"""Is the collection already packed?"""
1561
1568
return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1563
def pack(self, hint=None, clean_obsolete_packs=False):
1570
def pack(self, hint=None):
1564
1571
"""Pack the pack collection totally."""
1565
1572
self.ensure_loaded()
1566
1573
total_packs = len(self._names)
1582
1589
pack_operations[-1][1].append(pack)
1583
1590
self._execute_pack_operations(pack_operations, OptimisingPacker)
1585
if clean_obsolete_packs:
1586
self._clear_obsolete_packs()
1588
1592
def plan_autopack_combinations(self, existing_packs, pack_distribution):
1589
1593
"""Plan a pack operation.
1784
1788
:param return: None.
1786
1790
for pack in packs:
1788
pack.pack_transport.rename(pack.file_name(),
1789
'../obsolete_packs/' + pack.file_name())
1790
except (errors.PathError, errors.TransportError), e:
1791
# TODO: Should these be warnings or mutters?
1792
mutter("couldn't rename obsolete pack, skipping it:\n%s"
1791
pack.pack_transport.rename(pack.file_name(),
1792
'../obsolete_packs/' + pack.file_name())
1794
1793
# TODO: Probably needs to know all possible indices for this pack
1795
1794
# - or maybe list the directory and move all indices matching this
1796
1795
# name whether we recognize it or not?
1798
1797
if self.chk_index is not None:
1799
1798
suffixes.append('.cix')
1800
1799
for suffix in suffixes:
1802
self._index_transport.rename(pack.name + suffix,
1803
'../obsolete_packs/' + pack.name + suffix)
1804
except (errors.PathError, errors.TransportError), e:
1805
mutter("couldn't rename obsolete index, skipping it:\n%s"
1800
self._index_transport.rename(pack.name + suffix,
1801
'../obsolete_packs/' + pack.name + suffix)
1808
1803
def pack_distribution(self, total_revisions):
1809
1804
"""Generate a list of the number of revisions to put in each pack.
1835
1830
self._remove_pack_indices(pack)
1836
1831
self.packs.remove(pack)
1838
def _remove_pack_indices(self, pack, ignore_missing=False):
1839
"""Remove the indices for pack from the aggregated indices.
1841
:param ignore_missing: Suppress KeyErrors from calling remove_index.
1843
for index_type in Pack.index_definitions.keys():
1844
attr_name = index_type + '_index'
1845
aggregate_index = getattr(self, attr_name)
1846
if aggregate_index is not None:
1847
pack_index = getattr(pack, attr_name)
1849
aggregate_index.remove_index(pack_index)
1833
def _remove_pack_indices(self, pack):
1834
"""Remove the indices for pack from the aggregated indices."""
1835
self.revision_index.remove_index(pack.revision_index, pack)
1836
self.inventory_index.remove_index(pack.inventory_index, pack)
1837
self.text_index.remove_index(pack.text_index, pack)
1838
self.signature_index.remove_index(pack.signature_index, pack)
1839
if self.chk_index is not None:
1840
self.chk_index.remove_index(pack.chk_index, pack)
1855
1842
def reset(self):
1856
1843
"""Clear all cached data."""
1889
1876
disk_nodes = set()
1890
1877
for index, key, value in self._iter_disk_pack_index():
1891
1878
disk_nodes.add((key, value))
1892
orig_disk_nodes = set(disk_nodes)
1894
1880
# do a two-way diff against our original content
1895
1881
current_nodes = set()
1908
1894
disk_nodes.difference_update(deleted_nodes)
1909
1895
disk_nodes.update(new_nodes)
1911
return disk_nodes, deleted_nodes, new_nodes, orig_disk_nodes
1897
return disk_nodes, deleted_nodes, new_nodes
1913
1899
def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1914
1900
"""Given the correct set of pack files, update our saved info.
1954
1940
added.append(name)
1955
1941
return removed, added, modified
1957
def _save_pack_names(self, clear_obsolete_packs=False, obsolete_packs=None):
1943
def _save_pack_names(self, clear_obsolete_packs=False):
1958
1944
"""Save the list of packs.
1960
1946
This will take out the mutex around the pack names list for the
1965
1951
:param clear_obsolete_packs: If True, clear out the contents of the
1966
1952
obsolete_packs directory.
1967
:param obsolete_packs: Packs that are obsolete once the new pack-names
1968
file has been written.
1969
1953
:return: A list of the names saved that were not previously on disk.
1971
already_obsolete = []
1972
1955
self.lock_names()
1974
1957
builder = self._index_builder_class()
1975
(disk_nodes, deleted_nodes, new_nodes,
1976
orig_disk_nodes) = self._diff_pack_names()
1958
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
1977
1959
# TODO: handle same-name, index-size-changes here -
1978
1960
# e.g. use the value from disk, not ours, *unless* we're the one
1981
1963
builder.add_node(key, value)
1982
1964
self.transport.put_file('pack-names', builder.finish(),
1983
1965
mode=self.repo.bzrdir._get_file_mode())
1966
# move the baseline forward
1984
1967
self._packs_at_load = disk_nodes
1985
1968
if clear_obsolete_packs:
1988
to_preserve = set([o.name for o in obsolete_packs])
1989
already_obsolete = self._clear_obsolete_packs(to_preserve)
1969
self._clear_obsolete_packs()
1991
1971
self._unlock_names()
1992
1972
# synchronise the memory packs list with what we just wrote:
1993
1973
self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1995
# TODO: We could add one more condition here. "if o.name not in
1996
# orig_disk_nodes and o != the new_pack we haven't written to
1997
# disk yet. However, the new pack object is not easily
1998
# accessible here (it would have to be passed through the
1999
# autopacking code, etc.)
2000
obsolete_packs = [o for o in obsolete_packs
2001
if o.name not in already_obsolete]
2002
self._obsolete_packs(obsolete_packs)
2003
1974
return [new_node[0][0] for new_node in new_nodes]
2005
1976
def reload_pack_names(self):
2022
1993
# out the new value.
2023
(disk_nodes, deleted_nodes, new_nodes,
2024
orig_disk_nodes) = self._diff_pack_names()
1994
disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
2025
1995
# _packs_at_load is meant to be the explicit list of names in
2026
1996
# 'pack-names' at then start. As such, it should not contain any
2027
1997
# pending names that haven't been written out yet.
2028
self._packs_at_load = orig_disk_nodes
1998
pack_names_nodes = disk_nodes.difference(new_nodes)
1999
pack_names_nodes.update(deleted_nodes)
2000
self._packs_at_load = pack_names_nodes
2029
2001
(removed, added,
2030
2002
modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
2031
2003
if removed or added or modified:
2041
2013
raise errors.RetryAutopack(self.repo, False, sys.exc_info())
2043
def _clear_obsolete_packs(self, preserve=None):
2015
def _clear_obsolete_packs(self):
2044
2016
"""Delete everything from the obsolete-packs directory.
2046
:return: A list of pack identifiers (the filename without '.pack') that
2047
were found in obsolete_packs.
2050
2018
obsolete_pack_transport = self.transport.clone('obsolete_packs')
2051
if preserve is None:
2053
2019
for filename in obsolete_pack_transport.list_dir('.'):
2054
name, ext = osutils.splitext(filename)
2057
if name in preserve:
2060
2021
obsolete_pack_transport.delete(filename)
2061
2022
except (errors.PathError, errors.TransportError), e:
2062
warning("couldn't delete obsolete pack, skipping it:\n%s"
2023
warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
2066
2025
def _start_write_group(self):
2067
2026
# Do not permit preparation for writing if we're not in a 'write lock'.
2094
2053
# FIXME: just drop the transient index.
2095
2054
# forget what names there are
2096
2055
if self._new_pack is not None:
2097
operation = cleanup.OperationWithCleanups(self._new_pack.abort)
2098
operation.add_cleanup(setattr, self, '_new_pack', None)
2099
# If we aborted while in the middle of finishing the write
2100
# group, _remove_pack_indices could fail because the indexes are
2101
# already gone. But they're not there we shouldn't fail in this
2102
# case, so we pass ignore_missing=True.
2103
operation.add_cleanup(self._remove_pack_indices, self._new_pack,
2104
ignore_missing=True)
2105
operation.run_simple()
2057
self._new_pack.abort()
2059
# XXX: If we aborted while in the middle of finishing the write
2060
# group, _remove_pack_indices can fail because the indexes are
2061
# already gone. If they're not there we shouldn't fail in this
2062
# case. -- mbp 20081113
2063
self._remove_pack_indices(self._new_pack)
2064
self._new_pack = None
2106
2065
for resumed_pack in self._resumed_packs:
2107
operation = cleanup.OperationWithCleanups(resumed_pack.abort)
2108
# See comment in previous finally block.
2109
operation.add_cleanup(self._remove_pack_indices, resumed_pack,
2110
ignore_missing=True)
2111
operation.run_simple()
2067
resumed_pack.abort()
2069
# See comment in previous finally block.
2071
self._remove_pack_indices(resumed_pack)
2112
2074
del self._resumed_packs[:]
2114
2076
def _remove_resumed_pack_indices(self):
2378
2340
raise NotImplementedError(self.dont_leave_lock_in_place)
2380
2342
@needs_write_lock
2381
def pack(self, hint=None, clean_obsolete_packs=False):
2343
def pack(self, hint=None):
2382
2344
"""Compress the data within the repository.
2384
2346
This will pack all the data to a single pack. In future it may
2385
2347
recompress deltas or do other such expensive operations.
2387
self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2349
self._pack_collection.pack(hint=hint)
2389
2351
@needs_write_lock
2390
2352
def reconcile(self, other=None, thorough=False):
2546
2508
utf8_files = [('format', self.get_format_string())]
2548
2510
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2549
repository = self.open(a_bzrdir=a_bzrdir, _found=True)
2550
self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
2511
return self.open(a_bzrdir=a_bzrdir, _found=True)
2553
2513
def open(self, a_bzrdir, _found=False, _override_transport=None):
2554
2514
"""See RepositoryFormat.open().
2891
2850
repository_class = KnitPackRepository
2892
2851
_commit_builder_class = PackRootCommitBuilder
2893
2852
rich_root_data = True
2895
2853
supports_tree_reference = True
2896
2854
supports_external_lookups = True
2897
2855
# What index classes to use
added
removed
bzrlib/repofmt/pack_repo.py
