/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to setup.py

  • Committer: John Arbash Meinel
  • Date: 2010-08-05 16:27:35 UTC
  • mto: This revision was merged to the branch mainline in revision 5374.
  • Revision ID: john@arbash-meinel.com-20100805162735-172opvx34sr5gpbl
Find a case where we are wasting a bit of memory.

Specifically the 'build_details' tuple contains a lot of wasted references,
and we hold on to one of these for each record we are fetching.
And for something like 'bzr pack', that is all keys.

For just loading all text build details on my bzr+ repository, With:
locations = b.repository.texts._index.get_build_details(b.repository.texts.keys())
This drops the memory consumption from:
WorkingSize   77604KiB
 to
WorkingSize   64640KiB

Or around 10.6MB. I worked it out to a savings of about 80 bytes/record
on data that can have hundreds of thousands of records (in 32-bit).

Show diffs side-by-side

added added

removed removed

Lines of Context:
setup.py
125
125
                f = file(batch_path, "w")
126
126
                f.write(batch_str)
127
127
                f.close()
128
 
                print "Created:", batch_path
129
 
            except Exception, e:
130
 
                print "ERROR: Unable to create %s: %s" % (batch_path, e)
 
128
                print("Created: %s" % batch_path)
 
129
            except Exception:
 
130
                e = sys.exc_info()[1]
 
131
                print("ERROR: Unable to create %s: %s" % (batch_path, e))
131
132
 
132
133
    def _quoted_path(self, path):
133
134
        if ' ' in path:
171
172
        from Pyrex.Distutils import build_ext
172
173
        from Pyrex.Compiler.Version import version as pyrex_version
173
174
    except ImportError:
174
 
        print "No Pyrex, trying Cython..."
 
175
        print("No Pyrex, trying Cython...")
175
176
        from Cython.Distutils import build_ext
176
177
        from Cython.Compiler.Version import version as pyrex_version
177
178
except ImportError:
178
179
    have_pyrex = False
179
180
    # try to build the extension from the prior generated source.
180
 
    print
181
 
    print ("The python package 'Pyrex' is not available."
182
 
           " If the .c files are available,")
183
 
    print ("they will be built,"
184
 
           " but modifying the .pyx files will not rebuild them.")
185
 
    print
 
181
    print("")
 
182
    print("The python package 'Pyrex' is not available."
 
183
          " If the .c files are available,")
 
184
    print("they will be built,"
 
185
          " but modifying the .pyx files will not rebuild them.")
 
186
    print("")
186
187
    from distutils.command.build_ext import build_ext
187
188
else:
188
189
    have_pyrex = True
204
205
    def run(self):
205
206
        try:
206
207
            build_ext.run(self)
207
 
        except DistutilsPlatformError, e:
 
208
        except DistutilsPlatformError:
 
209
            e = sys.exc_info()[1]
208
210
            if not self.allow_python_fallback:
209
211
                log.warn('\n  Cannot build extensions.\n'
210
212
                         '  Use "build_ext --allow-python-fallback" to use'
281
283
    add_pyrex_extension('bzrlib._dirstate_helpers_pyx',
282
284
                        libraries=['Ws2_32'])
283
285
    add_pyrex_extension('bzrlib._walkdirs_win32')
284
 
    z_lib = 'zdll'
285
286
else:
286
287
    if have_pyrex and pyrex_version_info[:3] == (0,9,4):
287
288
        # Pyrex 0.9.4.1 fails to compile this extension correctly
290
291
        # which is NULL safe with PY_DECREF which is not.)
291
292
        # <https://bugs.edge.launchpad.net/bzr/+bug/449372>
292
293
        # <https://bugs.edge.launchpad.net/bzr/+bug/276868>
293
 
        print 'Cannot build extension "bzrlib._dirstate_helpers_pyx" using'
294
 
        print 'your version of pyrex "%s". Please upgrade your pyrex' % (
295
 
            pyrex_version,)
296
 
        print 'install. For now, the non-compiled (python) version will'
297
 
        print 'be used instead.'
 
294
        print('Cannot build extension "bzrlib._dirstate_helpers_pyx" using')
 
295
        print('your version of pyrex "%s". Please upgrade your pyrex' % (
 
296
            pyrex_version,))
 
297
        print('install. For now, the non-compiled (python) version will')
 
298
        print('be used instead.')
298
299
    else:
299
300
        add_pyrex_extension('bzrlib._dirstate_helpers_pyx')
300
301
    add_pyrex_extension('bzrlib._readdir_pyx')
301
 
    z_lib = 'z'
302
 
add_pyrex_extension('bzrlib._chk_map_pyx', libraries=[z_lib])
 
302
add_pyrex_extension('bzrlib._chk_map_pyx')
303
303
ext_modules.append(Extension('bzrlib._patiencediff_c',
304
304
                             ['bzrlib/_patiencediff_c.c']))
305
305
if have_pyrex and pyrex_version_info < (0, 9, 6, 3):
306
 
    print
307
 
    print 'Your Pyrex/Cython version %s is too old to build the simple_set' % (
308
 
        pyrex_version)
309
 
    print 'and static_tuple extensions.'
310
 
    print 'Please upgrade to at least Pyrex 0.9.6.3'
311
 
    print
 
306
    print("")
 
307
    print('Your Pyrex/Cython version %s is too old to build the simple_set' % (
 
308
        pyrex_version))
 
309
    print('and static_tuple extensions.')
 
310
    print('Please upgrade to at least Pyrex 0.9.6.3')
 
311
    print("")
312
312
    # TODO: Should this be a fatal error?
313
313
else:
314
314
    # We only need 0.9.6.3 to build _simple_set_pyx, but static_tuple depends
320
320
 
321
321
 
322
322
if unavailable_files:
323
 
    print 'C extension(s) not found:'
324
 
    print '   %s' % ('\n  '.join(unavailable_files),)
325
 
    print 'The python versions will be used instead.'
326
 
    print
 
323
    print('C extension(s) not found:')
 
324
    print('   %s' % ('\n  '.join(unavailable_files),))
 
325
    print('The python versions will be used instead.')
 
326
    print("")
327
327
 
328
328
 
329
329
def get_tbzr_py2exe_info(includes, excludes, packages, console_targets,
413
413
 
414
414
def get_qbzr_py2exe_info(includes, excludes, packages, data_files):
415
415
    # PyQt4 itself still escapes the plugin detection code for some reason...
416
 
    packages.append('PyQt4')
417
 
    excludes.append('PyQt4.elementtree.ElementTree')
418
 
    excludes.append('PyQt4.uic.port_v3')
 
416
    includes.append('PyQt4.QtCore')
 
417
    includes.append('PyQt4.QtGui')
419
418
    includes.append('sip') # extension module required for Qt.
420
419
    packages.append('pygments') # colorizer for qbzr
421
420
    packages.append('docutils') # html formatting
422
421
    includes.append('win32event')  # for qsubprocess stuff
423
 
    # but we can avoid many Qt4 Dlls.
424
 
    dll_excludes.extend(
425
 
        """QtAssistantClient4.dll QtCLucene4.dll QtDesigner4.dll
426
 
        QtHelp4.dll QtNetwork4.dll QtOpenGL4.dll QtScript4.dll
427
 
        QtSql4.dll QtTest4.dll QtWebKit4.dll QtXml4.dll
428
 
        qscintilla2.dll""".split())
429
422
    # the qt binaries might not be on PATH...
430
423
    # They seem to install to a place like C:\Python25\PyQt4\*
431
424
    # Which is not the same as C:\Python25\Lib\site-packages\PyQt4
534
527
            install_data.run(self)
535
528
 
536
529
            py2exe = self.distribution.get_command_obj('py2exe', False)
537
 
            optimize = py2exe.optimize
 
530
            # GZ 2010-04-19: Setup has py2exe.optimize as 2, but give plugins
 
531
            #                time before living with docstring stripping
 
532
            optimize = 1
538
533
            compile_names = [f for f in self.outfiles if f.endswith('.py')]
 
534
            # Round mtime to nearest even second so that installing on a FAT
 
535
            # filesystem bytecode internal and script timestamps will match
 
536
            for f in compile_names:
 
537
                mtime = os.stat(f).st_mtime
 
538
                remainder = mtime % 2
 
539
                if remainder:
 
540
                    mtime -= remainder
 
541
                    os.utime(f, (mtime, mtime))
539
542
            byte_compile(compile_names,
540
543
                         optimize=optimize,
541
544
                         force=self.force, prefix=self.install_dir,
542
545
                         dry_run=self.dry_run)
543
 
            if optimize:
544
 
                suffix = 'o'
545
 
            else:
546
 
                suffix = 'c'
547
 
            self.outfiles.extend([f + suffix for f in compile_names])
 
546
            self.outfiles.extend([f + 'o' for f in compile_names])
548
547
    # end of class install_data_with_bytecompile
549
548
 
550
549
    target = py2exe.build_exe.Target(script = "bzr",
681
680
        # print this warning to stderr as output is redirected, so it is seen
682
681
        # at build time.  Also to stdout so it appears in the log
683
682
        for f in (sys.stderr, sys.stdout):
684
 
            print >> f, \
685
 
                "Skipping TBZR binaries - please set TBZR to a directory to enable"
 
683
            f.write("Skipping TBZR binaries - "
 
684
                "please set TBZR to a directory to enable\n")
686
685
 
687
686
    # MSWSOCK.dll is a system-specific library, which py2exe accidentally pulls
688
687
    # in on Vista.
692
691
                               "excludes": excludes,
693
692
                               "dll_excludes": dll_excludes,
694
693
                               "dist_dir": "win32_bzr.exe",
695
 
                               "optimize": 1,
 
694
                               "optimize": 2,
696
695
                              },
697
696
                   }
698
697
 
699
 
    setup(options=options_list,
700
 
          console=console_targets,
701
 
          windows=gui_targets,
702
 
          zipfile='lib/library.zip',
703
 
          data_files=data_files,
704
 
          cmdclass={'install_data': install_data_with_bytecompile},
705
 
          )
 
698
    # We want the libaray.zip to have optimize = 2, but the exe to have
 
699
    # optimize = 1, so that .py files that get compilied at run time
 
700
    # (e.g. user installed plugins) dont have their doc strings removed.
 
701
    class py2exe_no_oo_exe(py2exe.build_exe.py2exe):
 
702
        def build_executable(self, *args, **kwargs):
 
703
            self.optimize = 1
 
704
            py2exe.build_exe.py2exe.build_executable(self, *args, **kwargs)
 
705
            self.optimize = 2
 
706
 
 
707
    if __name__ == '__main__':
 
708
        setup(options=options_list,
 
709
              console=console_targets,
 
710
              windows=gui_targets,
 
711
              zipfile='lib/library.zip',
 
712
              data_files=data_files,
 
713
              cmdclass={'install_data': install_data_with_bytecompile,
 
714
                        'py2exe': py2exe_no_oo_exe},
 
715
              )
706
716
 
707
717
else:
708
718
    # ad-hoc for easy_install
736
746
    ARGS.update(BZRLIB)
737
747
    ARGS.update(PKG_DATA)
738
748
 
739
 
    setup(**ARGS)
 
749
    if __name__ == '__main__':
 
750
        setup(**ARGS)