mirror of
https://github.com/nodejs/node.git
synced 2024-12-01 16:10:02 +01:00
d937b029a9
PR-URL: https://github.com/nodejs/node/pull/29213 Reviewed-By: Steven R Loomis <srloomis@us.ibm.com> Reviewed-By: Jiawen Geng <technicalcute@gmail.com> Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
357 lines
12 KiB
Python
Executable File
357 lines
12 KiB
Python
Executable File
#!/usr/bin/python
|
|
#
|
|
# Copyright (C) 2014 IBM Corporation and Others. All Rights Reserved.
|
|
#
|
|
# @author Steven R. Loomis <srl@icu-project.org>
|
|
#
|
|
# This tool slims down an ICU data (.dat) file according to a config file.
|
|
#
|
|
# See: http://bugs.icu-project.org/trac/ticket/10922
|
|
#
|
|
# Usage:
|
|
# Use "-h" to get help options.
|
|
|
|
from __future__ import print_function
|
|
|
|
import json
|
|
import optparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
|
|
try:
|
|
# for utf-8 on Python 2
|
|
reload(sys)
|
|
sys.setdefaultencoding("utf-8")
|
|
except NameError:
|
|
pass # Python 3 already defaults to utf-8
|
|
|
|
try:
|
|
basestring # Python 2
|
|
except NameError:
|
|
basestring = str # Python 3
|
|
|
|
endian=sys.byteorder
|
|
|
|
parser = optparse.OptionParser(usage="usage: mkdir tmp ; %prog -D ~/Downloads/icudt53l.dat -T tmp -F trim_en.json -O icudt53l.dat" )
|
|
|
|
parser.add_option("-P","--tool-path",
|
|
action="store",
|
|
dest="toolpath",
|
|
help="set the prefix directory for ICU tools")
|
|
|
|
parser.add_option("-D","--input-file",
|
|
action="store",
|
|
dest="datfile",
|
|
help="input data file (icudt__.dat)",
|
|
) # required
|
|
|
|
parser.add_option("-F","--filter-file",
|
|
action="store",
|
|
dest="filterfile",
|
|
help="filter file (JSON format)",
|
|
) # required
|
|
|
|
parser.add_option("-T","--tmp-dir",
|
|
action="store",
|
|
dest="tmpdir",
|
|
help="working directory.",
|
|
) # required
|
|
|
|
parser.add_option("--delete-tmp",
|
|
action="count",
|
|
dest="deltmpdir",
|
|
help="delete working directory.",
|
|
default=0)
|
|
|
|
parser.add_option("-O","--outfile",
|
|
action="store",
|
|
dest="outfile",
|
|
help="outfile (NOT a full path)",
|
|
) # required
|
|
|
|
parser.add_option("-v","--verbose",
|
|
action="count",
|
|
default=0)
|
|
|
|
parser.add_option('-L',"--locales",
|
|
action="store",
|
|
dest="locales",
|
|
help="sets the 'locales.only' variable",
|
|
default=None)
|
|
|
|
parser.add_option('-e', '--endian', action='store', dest='endian', help='endian, big, little or host, your default is "%s".' % endian, default=endian, metavar='endianness')
|
|
|
|
(options, args) = parser.parse_args()
|
|
|
|
optVars = vars(options)
|
|
|
|
for opt in [ "datfile", "filterfile", "tmpdir", "outfile" ]:
|
|
if optVars[opt] is None:
|
|
print("Missing required option: %s" % opt)
|
|
sys.exit(1)
|
|
|
|
if options.verbose>0:
|
|
print("Options: "+str(options))
|
|
|
|
if (os.path.isdir(options.tmpdir) and options.deltmpdir):
|
|
if options.verbose>1:
|
|
print("Deleting tmp dir %s.." % (options.tmpdir))
|
|
shutil.rmtree(options.tmpdir)
|
|
|
|
if not (os.path.isdir(options.tmpdir)):
|
|
os.mkdir(options.tmpdir)
|
|
else:
|
|
print("Please delete tmpdir %s before beginning." % options.tmpdir)
|
|
sys.exit(1)
|
|
|
|
if options.endian not in ("big","little","host"):
|
|
print("Unknown endianness: %s" % options.endian)
|
|
sys.exit(1)
|
|
|
|
if options.endian == "host":
|
|
options.endian = endian
|
|
|
|
if not os.path.isdir(options.tmpdir):
|
|
print("Error, tmpdir not a directory: %s" % (options.tmpdir))
|
|
sys.exit(1)
|
|
|
|
if not os.path.isfile(options.filterfile):
|
|
print("Filterfile doesn't exist: %s" % (options.filterfile))
|
|
sys.exit(1)
|
|
|
|
if not os.path.isfile(options.datfile):
|
|
print("Datfile doesn't exist: %s" % (options.datfile))
|
|
sys.exit(1)
|
|
|
|
if not options.datfile.endswith(".dat"):
|
|
print("Datfile doesn't end with .dat: %s" % (options.datfile))
|
|
sys.exit(1)
|
|
|
|
outfile = os.path.join(options.tmpdir, options.outfile)
|
|
|
|
if os.path.isfile(outfile):
|
|
print("Error, output file does exist: %s" % (outfile))
|
|
sys.exit(1)
|
|
|
|
if not options.outfile.endswith(".dat"):
|
|
print("Outfile doesn't end with .dat: %s" % (options.outfile))
|
|
sys.exit(1)
|
|
|
|
dataname=options.outfile[0:-4]
|
|
|
|
|
|
## TODO: need to improve this. Quotes, etc.
|
|
def runcmd(tool, cmd, doContinue=False):
|
|
if(options.toolpath):
|
|
cmd = os.path.join(options.toolpath, tool) + " " + cmd
|
|
else:
|
|
cmd = tool + " " + cmd
|
|
|
|
if(options.verbose>4):
|
|
print("# " + cmd)
|
|
|
|
rc = os.system(cmd)
|
|
if rc != 0 and not doContinue:
|
|
print("FAILED: %s" % cmd)
|
|
sys.exit(1)
|
|
return rc
|
|
|
|
## STEP 0 - read in json config
|
|
fi= open(options.filterfile, "rb")
|
|
config=json.load(fi)
|
|
fi.close()
|
|
|
|
if options.locales:
|
|
config["variables"] = config.get("variables", {})
|
|
config["variables"]["locales"] = config["variables"].get("locales", {})
|
|
config["variables"]["locales"]["only"] = options.locales.split(',')
|
|
|
|
if options.verbose > 6:
|
|
print(config)
|
|
|
|
if "comment" in config:
|
|
print("%s: %s" % (options.filterfile, config["comment"]))
|
|
|
|
## STEP 1 - copy the data file, swapping endianness
|
|
## The first letter of endian_letter will be 'b' or 'l' for big or little
|
|
endian_letter = options.endian[0]
|
|
|
|
runcmd("icupkg", "-t%s %s %s""" % (endian_letter, options.datfile, outfile))
|
|
|
|
## STEP 2 - get listing
|
|
listfile = os.path.join(options.tmpdir,"icudata.lst")
|
|
runcmd("icupkg", "-l %s > %s""" % (outfile, listfile))
|
|
|
|
with open(listfile, 'rb') as fi:
|
|
items = [line.strip() for line in fi.read().decode("utf-8").splitlines()]
|
|
itemset = set(items)
|
|
|
|
if options.verbose > 1:
|
|
print("input file: %d items" % len(items))
|
|
|
|
# list of all trees
|
|
trees = {}
|
|
RES_INDX = "res_index.res"
|
|
remove = None
|
|
# remove - always remove these
|
|
if "remove" in config:
|
|
remove = set(config["remove"])
|
|
else:
|
|
remove = set()
|
|
|
|
# keep - always keep these
|
|
if "keep" in config:
|
|
keep = set(config["keep"])
|
|
else:
|
|
keep = set()
|
|
|
|
def queueForRemoval(tree):
|
|
global remove
|
|
if tree not in config.get("trees", {}):
|
|
return
|
|
mytree = trees[tree]
|
|
if options.verbose > 0:
|
|
print("* %s: %d items" % (tree, len(mytree["locs"])))
|
|
# do varible substitution for this tree here
|
|
if isinstance(config["trees"][tree], basestring):
|
|
treeStr = config["trees"][tree]
|
|
if options.verbose > 5:
|
|
print(" Substituting $%s for tree %s" % (treeStr, tree))
|
|
if treeStr not in config.get("variables", {}):
|
|
print(" ERROR: no variable: variables.%s for tree %s" % (treeStr, tree))
|
|
sys.exit(1)
|
|
config["trees"][tree] = config["variables"][treeStr]
|
|
myconfig = config["trees"][tree]
|
|
if options.verbose > 4:
|
|
print(" Config: %s" % (myconfig))
|
|
# Process this tree
|
|
if(len(myconfig)==0 or len(mytree["locs"])==0):
|
|
if(options.verbose>2):
|
|
print(" No processing for %s - skipping" % (tree))
|
|
else:
|
|
only = None
|
|
if "only" in myconfig:
|
|
only = set(myconfig["only"])
|
|
if (len(only)==0) and (mytree["treeprefix"] != ""):
|
|
thePool = "%spool.res" % (mytree["treeprefix"])
|
|
if (thePool in itemset):
|
|
if(options.verbose>0):
|
|
print("Removing %s because tree %s is empty." % (thePool, tree))
|
|
remove.add(thePool)
|
|
else:
|
|
print("tree %s - no ONLY")
|
|
for l in range(len(mytree["locs"])):
|
|
loc = mytree["locs"][l]
|
|
if (only is not None) and not loc in only:
|
|
# REMOVE loc
|
|
toRemove = "%s%s%s" % (mytree["treeprefix"], loc, mytree["extension"])
|
|
if(options.verbose>6):
|
|
print("Queueing for removal: %s" % toRemove)
|
|
remove.add(toRemove)
|
|
|
|
def addTreeByType(tree, mytree):
|
|
if(options.verbose>1):
|
|
print("(considering %s): %s" % (tree, mytree))
|
|
trees[tree] = mytree
|
|
mytree["locs"]=[]
|
|
for i in range(len(items)):
|
|
item = items[i]
|
|
if item.startswith(mytree["treeprefix"]) and item.endswith(mytree["extension"]):
|
|
mytree["locs"].append(item[len(mytree["treeprefix"]):-4])
|
|
# now, process
|
|
queueForRemoval(tree)
|
|
|
|
addTreeByType("converters",{"treeprefix":"", "extension":".cnv"})
|
|
addTreeByType("stringprep",{"treeprefix":"", "extension":".spp"})
|
|
addTreeByType("translit",{"treeprefix":"translit/", "extension":".res"})
|
|
addTreeByType("brkfiles",{"treeprefix":"brkitr/", "extension":".brk"})
|
|
addTreeByType("brkdict",{"treeprefix":"brkitr/", "extension":"dict"})
|
|
addTreeByType("confusables",{"treeprefix":"", "extension":".cfu"})
|
|
|
|
for i in range(len(items)):
|
|
item = items[i]
|
|
if item.endswith(RES_INDX):
|
|
treeprefix = item[0:item.rindex(RES_INDX)]
|
|
tree = None
|
|
if treeprefix == "":
|
|
tree = "ROOT"
|
|
else:
|
|
tree = treeprefix[0:-1]
|
|
if(options.verbose>6):
|
|
print("procesing %s" % (tree))
|
|
trees[tree] = { "extension": ".res", "treeprefix": treeprefix, "hasIndex": True }
|
|
# read in the resource list for the tree
|
|
treelistfile = os.path.join(options.tmpdir,"%s.lst" % tree)
|
|
runcmd("iculslocs", "-i %s -N %s -T %s -l > %s" % (outfile, dataname, tree, treelistfile))
|
|
fi = open(treelistfile, 'rb')
|
|
treeitems = fi.readlines()
|
|
trees[tree]["locs"] = [treeitems[i].strip() for i in range(len(treeitems))]
|
|
fi.close()
|
|
if tree not in config.get("trees", {}):
|
|
print(" Warning: filter file %s does not mention trees.%s - will be kept as-is" % (options.filterfile, tree))
|
|
else:
|
|
queueForRemoval(tree)
|
|
|
|
def removeList(count=0):
|
|
# don't allow "keep" items to creep in here.
|
|
global remove
|
|
remove = remove - keep
|
|
if(count > 10):
|
|
print("Giving up - %dth attempt at removal." % count)
|
|
sys.exit(1)
|
|
if(options.verbose>1):
|
|
print("%d items to remove - try #%d" % (len(remove),count))
|
|
if(len(remove)>0):
|
|
oldcount = len(remove)
|
|
hackerrfile=os.path.join(options.tmpdir, "REMOVE.err")
|
|
removefile = os.path.join(options.tmpdir, "REMOVE.lst")
|
|
with open(removefile, 'wb') as fi:
|
|
fi.write('\n'.join(remove).encode("utf-8") + b'\n')
|
|
rc = runcmd("icupkg","-r %s %s 2> %s" % (removefile,outfile,hackerrfile),True)
|
|
if rc != 0:
|
|
if(options.verbose>5):
|
|
print("## Damage control, trying to parse stderr from icupkg..")
|
|
fi = open(hackerrfile, 'rb')
|
|
erritems = fi.readlines()
|
|
fi.close()
|
|
#Item zone/zh_Hant_TW.res depends on missing item zone/zh_Hant.res
|
|
pat = re.compile("""^Item ([^ ]+) depends on missing item ([^ ]+).*""")
|
|
for i in range(len(erritems)):
|
|
line = erritems[i].strip()
|
|
m = pat.match(line)
|
|
if m:
|
|
toDelete = m.group(1)
|
|
if(options.verbose > 5):
|
|
print("<< %s added to delete" % toDelete)
|
|
remove.add(toDelete)
|
|
else:
|
|
print("ERROR: could not match errline: %s" % line)
|
|
sys.exit(1)
|
|
if(options.verbose > 5):
|
|
print(" now %d items to remove" % len(remove))
|
|
if(oldcount == len(remove)):
|
|
print(" ERROR: could not add any mor eitems to remove. Fail.")
|
|
sys.exit(1)
|
|
removeList(count+1)
|
|
|
|
# fire it up
|
|
removeList(1)
|
|
|
|
# now, fixup res_index, one at a time
|
|
for tree in trees:
|
|
# skip trees that don't have res_index
|
|
if "hasIndex" not in trees[tree]:
|
|
continue
|
|
treebunddir = options.tmpdir
|
|
if(trees[tree]["treeprefix"]):
|
|
treebunddir = os.path.join(treebunddir, trees[tree]["treeprefix"])
|
|
if not (os.path.isdir(treebunddir)):
|
|
os.mkdir(treebunddir)
|
|
treebundres = os.path.join(treebunddir,RES_INDX)
|
|
treebundtxt = "%s.txt" % (treebundres[0:-4])
|
|
runcmd("iculslocs", "-i %s -N %s -T %s -b %s" % (outfile, dataname, tree, treebundtxt))
|
|
runcmd("genrb","-d %s -s %s res_index.txt" % (treebunddir, treebunddir))
|
|
runcmd("icupkg","-s %s -a %s%s %s" % (options.tmpdir, trees[tree]["treeprefix"], RES_INDX, outfile))
|