/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/transform.py

  • Committer: John Arbash Meinel
  • Date: 2010-08-05 16:27:35 UTC
  • mto: This revision was merged to the branch mainline in revision 5374.
  • Revision ID: john@arbash-meinel.com-20100805162735-172opvx34sr5gpbl
Find a case where we are wasting a bit of memory.

Specifically the 'build_details' tuple contains a lot of wasted references,
and we hold on to one of these for each record we are fetching.
And for something like 'bzr pack', that is all keys.

For just loading all text build details on my bzr+ repository, With:
locations = b.repository.texts._index.get_build_details(b.repository.texts.keys())
This drops the memory consumption from:
WorkingSize   77604KiB
 to
WorkingSize   64640KiB

Or around 10.6MB. I worked it out to a savings of about 80 bytes/record
on data that can have hundreds of thousands of records (in 32-bit).

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/transform.py
25
25
    annotate,
26
26
    bencode,
27
27
    bzrdir,
 
28
    commit,
28
29
    delta,
29
30
    errors,
30
31
    inventory,
927
928
        """
928
929
        return _PreviewTree(self)
929
930
 
930
 
    def commit(self, branch, message, merge_parents=None, strict=False):
 
931
    def commit(self, branch, message, merge_parents=None, strict=False,
 
932
               timestamp=None, timezone=None, committer=None, authors=None,
 
933
               revprops=None, revision_id=None):
931
934
        """Commit the result of this TreeTransform to a branch.
932
935
 
933
936
        :param branch: The branch to commit to.
934
937
        :param message: The message to attach to the commit.
935
 
        :param merge_parents: Additional parents specified by pending merges.
 
938
        :param merge_parents: Additional parent revision-ids specified by
 
939
            pending merges.
 
940
        :param strict: If True, abort the commit if there are unversioned
 
941
            files.
 
942
        :param timestamp: if not None, seconds-since-epoch for the time and
 
943
            date.  (May be a float.)
 
944
        :param timezone: Optional timezone for timestamp, as an offset in
 
945
            seconds.
 
946
        :param committer: Optional committer in email-id format.
 
947
            (e.g. "J Random Hacker <jrandom@example.com>")
 
948
        :param authors: Optional list of authors in email-id format.
 
949
        :param revprops: Optional dictionary of revision properties.
 
950
        :param revision_id: Optional revision id.  (Specifying a revision-id
 
951
            may reduce performance for some non-native formats.)
936
952
        :return: The revision_id of the revision committed.
937
953
        """
938
954
        self._check_malformed()
955
971
        if self._tree.get_revision_id() != last_rev_id:
956
972
            raise ValueError('TreeTransform not based on branch basis: %s' %
957
973
                             self._tree.get_revision_id())
958
 
        builder = branch.get_commit_builder(parent_ids)
 
974
        revprops = commit.Commit.update_revprops(revprops, branch, authors)
 
975
        builder = branch.get_commit_builder(parent_ids,
 
976
                                            timestamp=timestamp,
 
977
                                            timezone=timezone,
 
978
                                            committer=committer,
 
979
                                            revprops=revprops,
 
980
                                            revision_id=revision_id)
959
981
        preview = self.get_preview_tree()
960
982
        list(builder.record_iter_changes(preview, last_rev_id,
961
983
                                         self.iter_changes()))
1635
1657
                      or trans_id in self._new_parent):
1636
1658
                    try:
1637
1659
                        mover.rename(full_path, self._limbo_name(trans_id))
1638
 
                    except OSError, e:
 
1660
                    except errors.TransformRenameFailed, e:
1639
1661
                        if e.errno != errno.ENOENT:
1640
1662
                            raise
1641
1663
                    else:
1666
1688
                if trans_id in self._needs_rename:
1667
1689
                    try:
1668
1690
                        mover.rename(self._limbo_name(trans_id), full_path)
1669
 
                    except OSError, e:
 
1691
                    except errors.TransformRenameFailed, e:
1670
1692
                        # We may be renaming a dangling inventory id
1671
1693
                        if e.errno != errno.ENOENT:
1672
1694
                            raise
1770
1792
        parent_keys = [(file_id, self._file_revision(t, file_id)) for t in
1771
1793
                       self._iter_parent_trees()]
1772
1794
        vf.add_lines((file_id, tree_revision), parent_keys,
1773
 
                     self.get_file(file_id).readlines())
 
1795
                     self.get_file_lines(file_id))
1774
1796
        repo = self._get_repository()
1775
1797
        base_vf = repo.texts
1776
1798
        if base_vf not in vf.fallback_versionedfiles:
1798
1820
            executable = self.is_executable(file_id, path)
1799
1821
        return kind, executable, None
1800
1822
 
 
1823
    def is_locked(self):
 
1824
        return False
 
1825
 
1801
1826
    def lock_read(self):
1802
1827
        # Perhaps in theory, this should lock the TreeTransform?
1803
 
        pass
 
1828
        return self
1804
1829
 
1805
1830
    def unlock(self):
1806
1831
        pass
2266
2291
    for num, _unused in enumerate(wt.all_file_ids()):
2267
2292
        if num > 0:  # more than just a root
2268
2293
            raise errors.WorkingTreeAlreadyPopulated(base=wt.basedir)
2269
 
    existing_files = set()
2270
 
    for dir, files in wt.walkdirs():
2271
 
        existing_files.update(f[0] for f in files)
2272
2294
    file_trans_id = {}
2273
2295
    top_pb = bzrlib.ui.ui_factory.nested_progress_bar()
2274
2296
    pp = ProgressPhase("Build phase", 2, top_pb)
2298
2320
                precomputed_delta = []
2299
2321
            else:
2300
2322
                precomputed_delta = None
 
2323
            # Check if tree inventory has content. If so, we populate
 
2324
            # existing_files with the directory content. If there are no
 
2325
            # entries we skip populating existing_files as its not used.
 
2326
            # This improves performance and unncessary work on large
 
2327
            # directory trees. (#501307)
 
2328
            if total > 0:
 
2329
                existing_files = set()
 
2330
                for dir, files in wt.walkdirs():
 
2331
                    existing_files.update(f[0] for f in files)
2301
2332
            for num, (tree_path, entry) in \
2302
2333
                enumerate(tree.inventory.iter_entries_by_dir()):
2303
2334
                pb.update("Building tree", num - len(deferred_contents), total)
2435
2466
    if entry.kind == "directory":
2436
2467
        return True
2437
2468
    if entry.kind == "file":
2438
 
        if tree.get_file(file_id).read() == file(target_path, 'rb').read():
2439
 
            return True
 
2469
        f = file(target_path, 'rb')
 
2470
        try:
 
2471
            if tree.get_file_text(file_id) == f.read():
 
2472
                return True
 
2473
        finally:
 
2474
            f.close()
2440
2475
    elif entry.kind == "symlink":
2441
2476
        if tree.get_symlink_target(file_id) == os.readlink(target_path):
2442
2477
            return True
2494
2529
        raise errors.BadFileKindError(name, kind)
2495
2530
 
2496
2531
 
2497
 
@deprecated_function(deprecated_in((1, 9, 0)))
2498
 
def create_by_entry(tt, entry, tree, trans_id, lines=None, mode_id=None):
2499
 
    """Create new file contents according to an inventory entry.
2500
 
 
2501
 
    DEPRECATED.  Use create_from_tree instead.
2502
 
    """
2503
 
    if entry.kind == "file":
2504
 
        if lines is None:
2505
 
            lines = tree.get_file(entry.file_id).readlines()
2506
 
        tt.create_file(lines, trans_id, mode_id=mode_id)
2507
 
    elif entry.kind == "symlink":
2508
 
        tt.create_symlink(tree.get_symlink_target(entry.file_id), trans_id)
2509
 
    elif entry.kind == "directory":
2510
 
        tt.create_directory(trans_id)
2511
 
 
2512
 
 
2513
2532
def create_from_tree(tt, trans_id, tree, file_id, bytes=None,
2514
2533
    filter_tree_path=None):
2515
2534
    """Create new file contents according to tree contents.
2901
2920
        """Rename a file from one path to another."""
2902
2921
        try:
2903
2922
            osutils.rename(from_, to)
2904
 
        except OSError, e:
 
2923
        except (IOError, OSError), e:
2905
2924
            if e.errno in (errno.EEXIST, errno.ENOTEMPTY):
2906
2925
                raise errors.FileExists(to, str(e))
2907
 
            raise
 
2926
            # normal OSError doesn't include filenames so it's hard to see where
 
2927
            # the problem is, see https://bugs.launchpad.net/bzr/+bug/491763
 
2928
            raise errors.TransformRenameFailed(from_, to, str(e), e.errno)
2908
2929
        self.past_renames.append((from_, to))
2909
2930
 
2910
2931
    def pre_delete(self, from_, to):
2920
2941
    def rollback(self):
2921
2942
        """Reverse all renames that have been performed"""
2922
2943
        for from_, to in reversed(self.past_renames):
2923
 
            osutils.rename(to, from_)
 
2944
            try:
 
2945
                osutils.rename(to, from_)
 
2946
            except (OSError, IOError), e:
 
2947
                raise errors.TransformRenameFailed(to, from_, str(e), e.errno)                
2924
2948
        # after rollback, don't reuse _FileMover
2925
2949
        past_renames = None
2926
2950
        pending_deletions = None