/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/btree_index.py

  • Committer: Jelmer Vernooij
  • Date: 2009-03-28 14:24:46 UTC
  • mfrom: (4211 +trunk)
  • mto: This revision was merged to the branch mainline in revision 4212.
  • Revision ID: jelmer@samba.org-20090328142446-vqi0ksswdurga631
Merge bzr.dev.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/btree_index.py
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
#
17
17
 
18
18
"""B+Tree indices"""
30
30
    chunk_writer,
31
31
    debug,
32
32
    errors,
 
33
    fifo_cache,
33
34
    index,
34
35
    lru_cache,
35
36
    osutils,
180
181
        combine mem with the first and second indexes, creating a new one of
181
182
        size 4x. On the fifth create a single new one, etc.
182
183
        """
 
184
        if self._combine_backing_indices:
 
185
            (new_backing_file, size,
 
186
             backing_pos) = self._spill_mem_keys_and_combine()
 
187
        else:
 
188
            new_backing_file, size = self._spill_mem_keys_without_combining()
 
189
        dir_path, base_name = osutils.split(new_backing_file.name)
 
190
        # Note: The transport here isn't strictly needed, because we will use
 
191
        #       direct access to the new_backing._file object
 
192
        new_backing = BTreeGraphIndex(get_transport(dir_path),
 
193
                                      base_name, size)
 
194
        # GC will clean up the file
 
195
        new_backing._file = new_backing_file
 
196
        if self._combine_backing_indices:
 
197
            if len(self._backing_indices) == backing_pos:
 
198
                self._backing_indices.append(None)
 
199
            self._backing_indices[backing_pos] = new_backing
 
200
            for backing_pos in range(backing_pos):
 
201
                self._backing_indices[backing_pos] = None
 
202
        else:
 
203
            self._backing_indices.append(new_backing)
 
204
        self._keys = set()
 
205
        self._nodes = {}
 
206
        self._nodes_by_key = None
 
207
 
 
208
    def _spill_mem_keys_without_combining(self):
 
209
        return self._write_nodes(self._iter_mem_nodes(), allow_optimize=False)
 
210
 
 
211
    def _spill_mem_keys_and_combine(self):
183
212
        iterators_to_combine = [self._iter_mem_nodes()]
184
213
        pos = -1
185
214
        for pos, backing in enumerate(self._backing_indices):
191
220
        new_backing_file, size = \
192
221
            self._write_nodes(self._iter_smallest(iterators_to_combine),
193
222
                              allow_optimize=False)
194
 
        dir_path, base_name = osutils.split(new_backing_file.name)
195
 
        # Note: The transport here isn't strictly needed, because we will use
196
 
        #       direct access to the new_backing._file object
197
 
        new_backing = BTreeGraphIndex(get_transport(dir_path),
198
 
                                      base_name, size)
199
 
        # GC will clean up the file
200
 
        new_backing._file = new_backing_file
201
 
        if len(self._backing_indices) == backing_pos:
202
 
            self._backing_indices.append(None)
203
 
        self._backing_indices[backing_pos] = new_backing
204
 
        for pos in range(backing_pos):
205
 
            self._backing_indices[pos] = None
206
 
        self._keys = set()
207
 
        self._nodes = {}
208
 
        self._nodes_by_key = None
 
223
        return new_backing_file, size, backing_pos
209
224
 
210
225
    def add_nodes(self, nodes):
211
226
        """Add nodes to the index.
627
642
        # Default max size is 100,000 leave values
628
643
        self._leaf_value_cache = None # lru_cache.LRUCache(100*1000)
629
644
        self._leaf_node_cache = lru_cache.LRUCache(_NODE_CACHE_SIZE)
630
 
        self._internal_node_cache = lru_cache.LRUCache()
 
645
        # We could limit this, but even a 300k record btree has only 3k leaf
 
646
        # nodes, and only 20 internal nodes. So the default of 100 nodes in an
 
647
        # LRU would mean we always cache everything anyway, no need to pay the
 
648
        # overhead of LRU
 
649
        self._internal_node_cache = fifo_cache.FIFOCache(100)
631
650
        self._key_count = None
632
651
        self._row_lengths = None
633
652
        self._row_offsets = None # Start of each row, [-1] is the end