User:PhiLiP/ZhConversion.py

# -*- coding: utf-8  -*-
# file: ZhConversion.py

import re
import zipfile
import codecs

def getUnihan(unihan_fname='Unihan.zip'):
	try:
		unihanzipfile = zipfile.ZipFile(unihan_fname, 'r')
		data = unihanzipfile.read('Unihan.txt')
		print('Unihan.zip found.')
	except IOError:
		print('Unihan.zip not found.')
		return False
	return data

def getConversionTable(ctable_fname):
	try:
		conversiontable = codecs.open(ctable_fname, 'r', 'utf-8')
		data = conversiontable.read()
		print(ctable_fname + ' found.')
	except IOError:
		print(ctable_fname + ' not found.')
		return False
	return data

def getConversionTableDiff(ctablediff_fname):
	try:
		conversiontablediff = codecs.open(ctablediff_fname + '.diff', 'r', 'utf-8')
		data = conversiontablediff.read()
	except IOError:
		data = False
	return data

def patchConversionTable(orig, diff, variant):
	print('  ' + variant + ".diff found, try to merge with file " + variant + '...')
	origlines = orig.splitlines()
	origlines[0] = origlines[0].replace(u'\ufeff', '')
	difflines = diff.splitlines()
	i = 0
	for diffline in difflines:
		if diffline.startswith('@@'):
			i = int(diffline.split()[2].split(',')[0]) - 1
		elif diffline.startswith(' '):
			if diffline[1:] == origlines[i]:
				i += 1
			else:
				print('  ' + variant + ".diff can't merge with file " + variant + '.')
				return False
		elif diffline.startswith('+') and not diffline.startswith('+++'):
			origlines.insert(i, diffline[1:])
			i += 1
		elif diffline.startswith('-') and not diffline.startswith('---'):
			if diffline[1:] == origlines[i]:
				origlines.pop(i)
			else:
				print('  ' + variant + ".diff can't merge with file " + variant + '.')
				return False
	orig = '\n'.join(origlines)
	return orig

def getDictFromUnihan(variant):
	unihanfile = getUnihan(unihan_fname='Unihan.zip')
	elems = unihanfile.splitlines()
	to = {}
	sept = '\t' + variant + '\t'
	for elem in elems:
		left, sep, right = elem.partition(sept)
		if sep == sept:
			right = right.split()
			right = right[0]
			if left != right:
				to[ucs4chr(int(left[2:],16))] = ucs4chr(int(right[2:],16))
	return to

def getDictFromConversionTable(to, variant):
	conversiontable = getConversionTable(ctable_fname = variant)
	conversiontablediff = getConversionTableDiff(ctablediff_fname = variant)
	if conversiontablediff:
		conversiontb = patchConversionTable(conversiontable, conversiontablediff, variant)
		if conversiontb:
			saveConversionTable(variant, conversiontb)
			conversiontable = conversiontb
	p = re.compile('-\{([\s\S]*?)\}-')
	conversionslist = p.findall(conversiontable)
	elems = []
	for conversions in conversionslist:
		elems += conversions.splitlines()
	for elem in elems:
		left, sep, right = elem.partition('=>')
		if sep == '=>':
			left = left.replace('*','').strip()
			right = right.partition('//')[0].replace(';','').strip()
			if left in to:
				if left == right:
					to.pop(left)
				else:
					to[left] = right
			else:
				to[left] = right
	return to

def toHansDict():
	toHans = getDictFromUnihan('kSimplifiedVariant')
	toHans = getDictFromConversionTable(toHans, 'Zh-hans')
	return toHans

def toHantDict():
	toHant = getDictFromUnihan('kTraditionalVariant')
	toHant = getDictFromConversionTable(toHant, 'Zh-hant')
	return toHant

def toOtherDict(variant):
	toOther = {}
	toOther = getDictFromConversionTable(toOther, variant)
	return toOther

def getConversionCode(to):
	CString = ''
	for left, right in sorted(to.items(), key=lambda d: d[0]):
		CString += '"' + left + '" => "' + right + '",\n'
	return CString

def saveFile(toHant, toHans, toTW, toHK, toCN, toSG):
	CString = u'<?php\n/**\n * Simplified / Traditional Chinese conversion tables\n *' \
 			+ u'\n * Automatically generated using code and data in includes/zhtable/' \
 			+ u'\n * Do not modify directly!\n */\n\n'
	zhConversion = codecs.open('ZhConversion.php', 'w', 'utf-8')
	CString += u'$zh2Hant = array(\n'
	CString += getConversionCode(toHant)
	CString += u');\n\n$zh2Hans = array(\n'
	CString += getConversionCode(toHans)
	CString += u');\n\n$zh2TW = array(\n'
	CString += getConversionCode(toTW)
	CString += u');\n\n$zh2HK = array(\n'
	CString += getConversionCode(toHK)
	CString += u');\n\n$zh2CN = array(\n'
	CString += getConversionCode(toCN)
	CString += u');\n\n$zh2SG = array(\n'
	CString += getConversionCode(toSG)
	CString += u');'
	zhConversion.write(CString)
	print ('ZhConversion.php created / updated successfully.')
	zhConversion.close()

def saveConversionTable(variant, conversiontable):
	conversiontablefile = codecs.open(variant + '_new', 'w', 'utf-8')
	conversiontablefile.write(conversiontable)
	print ('  ' + variant + '_new created.')
	conversiontablefile.close()

def ucs4chr(codepoint):
	try:
		return unichr(codepoint)
	except ValueError:
		hi, lo = divmod (codepoint-0x10000, 0x400)
		return unichr(0xd800+hi) + unichr(0xdc00+lo)

def ucs4ord(str):
	if len(str)==1:
		return ord(str)
	if len(str)==2:
		hi, lo = ord(str[0])-0xd800, ord(str[1])-0xdc00
		return hi*0x400+0x10000
	raise TypeError("ucs4ord() expected a valid ucs4 character")

toHant = toHantDict()
toHans = toHansDict()
toTW = toOtherDict('Zh-tw')
toHK = toOtherDict('Zh-hk')
toCN = toOtherDict('Zh-cn')
toSG = toOtherDict('Zh-sg')
saveFile(toHant, toHans, toTW, toHK, toCN, toSG)