cpython/Tools/perfecthash/GenUCNHash.py

#! /usr/bin/env python
import sys
import string
import perfect_hash

# This is a user of perfect_hash.py
# that takes as input the UnicodeData.txt file available from:
# ftp://ftp.unicode.org/Public/UNIDATA/UnicodeData.txt

# It generates a hash table from Unicode Character Name ->
# unicode code space value.

# These variables determine which hash function is tried first.
# Yields a multiple of 1.7875 for UnicodeData.txt on 2000/06/24/
f1Seed = 1694245428
f2Seed = -1917331657

# Maximum allowed multipler, if this isn't None then instead of continually
# increasing C, it resets it back to initC to keep searching for
# a solution.
minC = 1.7875
# Initial multiplier for trying to find a perfect hash function.
initC = 1.7875

moduleName = "ucnhash"
dataArrayName = "aucn"
dataArrayType = "_Py_UnicodeCharacterName"
headerFileName = "ucnhash.h"
cFileName      = "ucnhash.c"
structName     = "_Py_UCNHashAPI"

keys = []
hashData = {}

def generateOutputFiles(perfHash, hashData):
  header = perfHash.generate_header(structName)
  header = header + """
typedef struct 
{
    const char *pszUCN;
    unsigned int uiValue;
} _Py_UnicodeCharacterName;

"""
  
  code = perfHash.generate_code(moduleName,
                                dataArrayName,
                                dataArrayType,
                                structName)
  out = open(headerFileName, "w")
  out.write(header)
  out = open(cFileName, "w")
  out.write("#include \"%s\"\n" % headerFileName)
  out.write(code)
  perfHash.generate_graph(out)
  out.write("""
  
static const _Py_UnicodeCharacterName aucn[] = 
{
""")
  for i in xrange(len(keys)):
    v = hashData[keys[i][0]]
    out.write('  { "' + keys[i][0] + '", ' + hex(v) + " }," + "\n")
  out.write("};\n\n")
  sys.stderr.write('\nGenerated output files: \n')
  sys.stderr.write('%s\n%s\n' % (headerFileName, cFileName))

def main():
  # Suck in UnicodeData.txt and spit out the generated files.
  input = open(sys.argv[1], 'r')
  i = 0
  while 1:
    line = input.readline()
    if line == "": break
    fields = string.split(line, ';')
    if len(fields) < 2:
      sys.stderr.write('Ill-formated line!\n')
      sys.stderr.write('line #: %d\n' % (i + 1))
      sys.exit()
    data, key = fields[:2]
    key = string.strip( key )
    # Any name starting with '<' is a control, or start/end character,
    # so skip it...
    if key[0] == "<":
      continue
    hashcode = i
    i = i + 1
    # force the name to uppercase
    keys.append( (string.upper(key),hashcode) )
    data = string.atoi(data, 16)
    hashData[key] = data

  input.close()
  sys.stderr.write('%i key/hash pairs read\n' % len(keys) )
  perfHash = perfect_hash.generate_hash(keys, 1,
                                        minC, initC,
                                        f1Seed, f2Seed,
                                        # increment, tries
                                        0.0025, 50)
  generateOutputFiles(perfHash, hashData)

if __name__ == '__main__':
  if len(sys.argv) == 1:
    sys.stdout = sys.stderr
    print 'Usage: %s <input filename>' % sys.argv[0]
    print '  The input file needs to be UnicodeData.txt'
    sys.exit()
  main()