#! /usr/local/bin/python # Given a Python script, create a binary that runs the script. # The binary is 100% independent of Python libraries and binaries. # It will not contain any Python source code -- only "compiled" Python # (as initialized static variables containing marshalled code objects). # It even does the right thing for dynamically loaded modules! # The module search path of the binary is set to the current directory. # # Some problems remain: # - You need to have the Python source tree lying around as well as # the various libraries used to generate the Python binary. # - For scripts that use many modules it generates absurdly large # files (frozen.c and config.o as well as the final binary), # and is consequently rather slow. # # Caveats: # - The search for modules sometimes finds modules that are never # actually imported since the code importing them is never executed. # - If an imported module isn't found, you get a warning but the # process of freezing continues. The binary will fail if it # actually tries to import one of these modules. # - This often happens with the module 'mac', which module 'os' tries # to import (to determine whether it is running on a Macintosh). # You can ignore the warning about this. # - If the program dynamically reads or generates Python code and # executes it, this code may reference built-in or library modules # that aren't present in the frozen binary, and this will fail. # - Your program may be using external data files, e.g. compiled # forms definitions (*.fd). These aren't incorporated. By default, # the sys.path in the resulting binary is only '.' (but you can override # that with the -P option). # # Usage hints: # - If you have a bunch of scripts that you want to freeze, instead # of freezing each of them separately, you might consider writing # a tiny main script that looks at sys.argv[0] and then imports # the corresponding module. You can then make links to the # frozen binary named after the various scripts you support. # Pass the additional scripts as arguments after the main script. # A minimal script to do this is the following. # import sys, posixpath # exec('import ' + posixpath.basename(sys.argv[0]) + '\n') # # Mods by Jack, August 94: # - Removed all static configuration stuff. Now, Setup and Makefile files # are parsed to obtain the linking info for the libraries. You have to # supply the -B option, though. # - Added -P (set sys.path) and -I/-D/-L/-l options (passed on to cc and # ld). import os import sys import regex import getopt import regsub import string import marshal # Exception used when scanfile fails NoSuchFile = 'NoSuchFile' # Global options builddir = '' # -B dir quiet = 0 # -q verbose = 0 # -v noexec = 0 # -n nowrite = 0 # -N ofile = 'a.out' # -o file path = '\'"."\'' # -P path cc_options = [] # Collects cc options ld_options = [] # Collects ld options module_libraries = {} # ld options for each module global_libraries = [] # Libraries we always need include_path = '' # Include path, from Makefile lib_path = '' # and lib path, ditto compiler = 'cc' # and compiler # Main program -- argument parsing etc. def main(): global quiet, verbose, noexec, nowrite, ofile, builddir, path try: opts, args = getopt.getopt(sys.argv[1:], 'B:nNo:P:qvI:D:L:l:') except getopt.error, msg: usage(str(msg)) sys.exit(2) for o, a in opts: if o == '-B': builddir = a if o == '-n': noexec = 1 if o == '-N': nowrite = 1 if o == '-o': ofile = a if o == '-P': if '"' in a: usage('sorry, cannot have " in -P option') sys.exit(2) path = `'"' + a + '"'` if o == '-q': verbose = 0; quiet = 1 if o == '-v': verbose = verbose + 1; quiet = 0 if o in ('-I', '-D'): cc_options.append(o+a) if o in ('-L', '-l'): ld_options.append(o+a) if not builddir: usage('sorry, you have to pass a -B option') sys.exit(2) if len(args) < 1: usage('please pass at least one file argument') sys.exit(2) process(args[0], args[1:]) # Print usage message to stderr def usage(*msgs): sys.stdout = sys.stderr for msg in msgs: print msg print 'Usage: freeze [options] scriptfile [modulefile ...]' print '-B dir : name of python build dir (no default)' print '-n : generate the files but don\'t compile and link' print '-N : don\'t write frozen.c (do compile unless -n given)' print '-o file : binary output file (default a.out)' print '-P path : set sys.path for program (default ".")' print '-q : quiet (no messages at all except errors)' print '-v : verbose (lots of extra messages)' print '-D and -I options are passed to cc, -L and -l to ld' # Process the script file def process(filename, addmodules): global noexec # if not quiet: print 'Computing needed modules ...' todo = {} todo['__main__'] = filename for name in addmodules: mod = os.path.basename(name) if mod[-3:] == '.py': mod = mod[:-3] todo[mod] = name try: dict = closure(todo) except NoSuchFile, filename: sys.stderr.write('Can\'t open file %s\n' % filename) sys.exit(1) # mods = dict.keys() mods.sort() # if verbose: print '%-15s %s' % ('Module', 'Filename') for mod in mods: print '%-15s %s' % (`mod`, dict[mod]) # if not quiet: print 'Looking for dynamically linked modules ...' dlmodules = [] objs = [] libs = [] for mod in mods: if dict[mod][-2:] == '.o': if verbose: print 'Found', mod, dict[mod] dlmodules.append(mod) objs.append(dict[mod]) libsname = dict[mod][:-2] + '.libs' try: f = open(libsname, 'r') except IOError: f = None if f: libtext = f.read() f.close() for lib in string.split(libtext): if lib in libs: libs.remove(lib) libs.append(lib) # if not nowrite: if not quiet: print 'Writing frozen.c ...' writefrozen('frozen.c', dict) else: if not quiet: print 'NOT writing frozen.c ...' # if not quiet: print 'Deducing compile/link options from', builddir # # Parse the config info # parse(builddir) CONFIG_IN = lib_path + '/config.c.in' FMAIN = lib_path + '/frozenmain.c' CC = compiler # ## if not dlmodules: if 0: config = CONFIG if not quiet: print 'Using existing', config, '...' else: config = 'tmpconfig.c' if nowrite: if not quiet: print 'NOT writing config.c ...' else: if not quiet: print 'Writing config.c with dl modules ...' f = open(CONFIG_IN, 'r') g = open(config, 'w') m1 = regex.compile('-- ADDMODULE MARKER 1 --') m2 = regex.compile('-- ADDMODULE MARKER 2 --') builtinmodules = [] stdmodules = ('sys', '__main__', '__builtin__', 'marshal') for mod in dict.keys(): if dict[mod] == '' and \ mod not in stdmodules: builtinmodules.append(mod) todomodules = builtinmodules + dlmodules while 1: line = f.readline() if not line: break g.write(line) if m1.search(line) >= 0: if verbose: print 'Marker 1 ...' for mod in todomodules: g.write('extern void init' + \ mod + '();\n') if m2.search(line) >= 0: if verbose: print 'Marker 2 ...' for mod in todomodules: g.write('{"' + mod + \ '", init' + mod + '},\n') g.close() # if not quiet: if noexec: print 'Generating compilation commands ...' else: print 'Starting compilation ...' defs = ['-DNO_MAIN', '-DUSE_FROZEN'] defs.append('-DPYTHONPATH='+path) # incs = ['-I.', '-I' + include_path] # if dict.has_key('stdwin'): # incs.append('-I' + j(STDWIN, 'H')) # srcs = [config, FMAIN] # modlibs = module_libraries for mod in dict.keys(): if modlibs.has_key(mod): libs = libs + modlibs[mod] libs = libs + global_libraries # # remove dups: # XXXX Not well tested... nskip = 0 newlibs = [] while libs: l = libs[0] del libs[0] if l[:2] == '-L' and l in newlibs: nskip = nskip + 1 continue if (l[:2] == '-l' or l[-2:] == '.a') and l in libs: nskip = nskip + 1 continue newlibs.append(l) libs = newlibs if nskip and not quiet: print 'Removed %d duplicate libraries'%nskip # sts = 0 # cmd = CC + ' -c' if cc_options: cmd = cmd + ' ' + string.join(cc_options) cmd = cmd + ' ' + string.join(defs) cmd = cmd + ' ' + string.join(incs) cmd = cmd + ' ' + string.join(srcs) print cmd # if not noexec: sts = os.system(cmd) if sts: print 'Exit status', sts, '-- turning on -n' noexec = 1 # for s in srcs: s = os.path.basename(s) if s[-2:] == '.c': s = s[:-2] o = s + '.o' objs.insert(0, o) # cmd = CC cmd = cmd + ' ' + string.join(objs) cmd = cmd + ' ' + string.join(libs) if ld_options: cmd = cmd + ' ' + string.join(ld_options) cmd = cmd + ' -o ' + ofile print cmd # if not noexec: sts = os.system(cmd) if sts: print 'Exit status', sts else: print 'Done.' # if not quiet and not noexec and sts == 0: print 'Note: consider this:'; print '\tstrip', ofile # sys.exit(sts) # Generate code for a given module def makecode(filename): if filename[-2:] == '.o': return None try: f = open(filename, 'r') except IOError: return None if verbose: print 'Making code from', filename, '...' text = f.read() code = compile(text, filename, 'exec') f.close() return marshal.dumps(code) # Write the C source file containing the frozen Python code def writefrozen(filename, dict): f = open(filename, 'w') codelist = [] for mod in dict.keys(): codestring = makecode(dict[mod]) if codestring is not None: codelist.append((mod, codestring)) write = sys.stdout.write save_stdout = sys.stdout try: sys.stdout = f for mod, codestring in codelist: if verbose: write('Writing initializer for %s\n'%mod) print 'static unsigned char M_' + mod + '[' + \ str(len(codestring)) + '+1] = {' for i in range(0, len(codestring), 16): for c in codestring[i:i+16]: print str(ord(c)) + ',', print print '};' print 'struct frozen {' print ' char *name;' print ' unsigned char *code;' print ' int size;' print '} frozen_modules[] = {' for mod, codestring in codelist: print ' {"' + mod + '",', print 'M_' + mod + ',', print str(len(codestring)) + '},' print ' {0, 0, 0} /* sentinel */' print '};' finally: sys.stdout = save_stdout f.close() # Determine the names and filenames of the modules imported by the # script, recursively. This is done by scanning for lines containing # import statements. (The scanning has only superficial knowledge of # Python syntax and no knowledge of semantics, so in theory the result # may be incorrect -- however this is quite unlikely if you don't # intentionally obscure your Python code.) # Compute the closure of scanfile() -- special first file because of script def closure(todo): done = {} while todo: newtodo = {} for modname in todo.keys(): if not done.has_key(modname): filename = todo[modname] if filename is None: filename = findmodule(modname) done[modname] = filename if filename in ('', ''): continue modules = scanfile(filename) for m in modules: if not done.has_key(m): newtodo[m] = None todo = newtodo return done # Scan a file looking for import statements importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)' fromstr = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+' isimport = regex.compile(importstr) isfrom = regex.compile(fromstr) def scanfile(filename): allmodules = {} try: f = open(filename, 'r') except IOError, msg: raise NoSuchFile, filename while 1: line = f.readline() if not line: break # EOF while line[-2:] == '\\\n': # Continuation line line = line[:-2] + ' ' line = line + f.readline() if isimport.search(line) >= 0: rawmodules = isimport.group(2) modules = string.splitfields(rawmodules, ',') for i in range(len(modules)): modules[i] = string.strip(modules[i]) elif isfrom.search(line) >= 0: modules = [isfrom.group(2)] else: continue for mod in modules: allmodules[mod] = None f.close() return allmodules.keys() # Find the file containing a module, given its name; None if not found builtins = sys.builtin_module_names + ['sys'] def findmodule(modname): if modname in builtins: return '' for dirname in sys.path: dlfullname = os.path.join(dirname, modname + 'module.o') try: f = open(dlfullname, 'r') except IOError: f = None if f: f.close() return dlfullname fullname = os.path.join(dirname, modname + '.py') try: f = open(fullname, 'r') except IOError: continue f.close() return fullname if not quiet: sys.stderr.write('Warning: module %s not found\n' % modname) return '' # # Parse a setup file. Returns two dictionaries, one containing variables # defined with their values and one containing module definitions # def parse_setup(fp): modules = {} variables = {} for line in fp.readlines(): if '#' in line: # Strip comments line = string.splitfields(line, '#')[0] line = string.strip(line[:-1]) # Strip whitespace if not line: continue words = string.split(line) if '=' in words[0]: # # equal sign before first space. Definition # pos = string.index(line, '=') name = line[:pos] value = string.strip(line[pos+1:]) variables[name] = value else: modules[words[0]] = words[1:] return modules, variables # # Parse a makefile. Returns a list of the variables defined. # def parse_makefile(fp): variables = {} for line in fp.readlines(): if '#' in line: # Strip comments line = string.splitfields(line, '#')[0] if not line: continue if line[0] in string.whitespace: continue line = string.strip(line[:-1]) # Strip whitespace if not line: continue if '=' in string.splitfields(line, ':')[0]: # # equal sign before first colon. Definition # pos = string.index(line, '=') name = line[:pos] value = string.strip(line[pos+1:]) variables[name] = value return variables # # Recursively add loader options from Setup files in extension # directories. # def add_extension_directory(name, isinstalldir): if verbose: print 'Adding extension directory', name fp = open(name + '/Setup', 'r') modules, variables = parse_setup(fp) # # Locate all new modules and remember the ld flags needed for them # for m in modules.keys(): if module_libraries.has_key(m): continue options = modules[m] if isinstalldir: ld_options = [] else: ld_options = [name + '/lib.a'] for o in options: # ld options are all capital except DUIC and l if o[:-2] == '.a': ld_options.append(o) elif o[0] == '-': if o[1] == 'l': ld_options.append(o) elif o[1] in string.uppercase and not o[1] in 'DUIC': ld_options.append(o) module_libraries[m] = ld_options # # See if we have to bother with base setups # if variables.has_key('BASESETUP'): if isinstalldir: raise 'installdir has base setup' setupfiles = string.split(variables['BASESETUP']) for s in setupfiles: if s[-6:] <> '/Setup': raise 'Incorrect BASESETUP', s s = s[:-6] if s[0] <> '/': s = name + '/' + s s = os.path.normpath(s) add_extension_directory(s, 0) # # Main routine for this module: given a build directory, get all # information needed for the linker. # def parse(dir): global include_path global lib_path global compiler fp = open(dir + '/Makefile', 'r') # # First find the global libraries and the base python # vars = parse_makefile(fp) if vars.has_key('CC'): compiler = vars['CC'] if not vars.has_key('installdir'): raise 'No $installdir in Makefile' include_path = vars['installdir'] + '/include/Py' lib_path = vars['installdir'] + '/lib/python/lib' global_libraries.append('-L' + lib_path) global_libraries.append('-lPython') global_libraries.append('-lParser') global_libraries.append('-lObjects') global_libraries.append('-lModules') for name in ('LIBS', 'LIBM', 'LIBC'): if not vars.has_key(name): raise 'Missing required def in Makefile', name for lib in string.split(vars[name]): global_libraries.append(lib) # # Next, parse the modules from the base python # add_extension_directory(lib_path, 1) # # Finally, parse the modules from the extension python # if dir <> lib_path: add_extension_directory(dir, 0) # Call the main program main()