#! /usr/bin/env python # -*- coding: utf-8 -*- #====================================================================== # # dictutils.py - # # Created by skywind on 2017/03/31 # Last change: 2017/03/31 22:20:13 # #====================================================================== import sys import os import time import stardict import codecs #---------------------------------------------------------------------- # python3 compatible #---------------------------------------------------------------------- if sys.version_info[0] >= 3: unicode = str long = int xrange = range #---------------------------------------------------------------------- # Word Generator #---------------------------------------------------------------------- class Generator (object): def __init__ (self): terms = {} terms['zk'] = u'中' terms['gk'] = u'高' terms['ky'] = u'研' terms['cet4'] = u'四' terms['cet6'] = u'六' terms['toefl'] = u'托' terms['ielts'] = u'雅' terms['gre'] = u'宝' self._terms = terms names = ('zk', 'gk', 'ky', 'cet4', 'cet6', 'toefl', 'ielts', 'gre') self._term_name = names def word_tag (self, data): tag = data.get('tag', '') text = '' for term in self._term_name: if not tag: continue if not term in tag: continue text += self._terms[term] frq = data.get('frq') if isinstance(frq, str) or isinstance(frq, unicode): if frq in ('', '0'): frq = None if not frq: frq = '-' bnc = data.get('bnc') if isinstance(bnc, str) or isinstance(bnc, unicode): if bnc in ('', '0'): bnc = None if not bnc: bnc = '-' if bnc != '-' or frq != '-': text += ' %s/%s'%(frq, bnc) return text.strip() def word_level (self, data): head = '' collins = data.get('collins', '') if isinstance(collins, str) or isinstance(collins, unicode): if collins in ('', '0'): collins = None if collins: head = str(collins) if data.get('oxford'): head = 'K' + head return head.strip() def word_exchange (self, data, style): if not data: return '' exchange = data.get('exchange') exchange = stardict.tools.exchange_loads(exchange) if not exchange: return '' part = [] last = '' count = 0 for k in ('p', 'd', 'i', '3'): p = exchange.get(k) if p: count += 1 if p != last: part.append(u'%s'%p) last = p if count < 4: text = '' else: text = ', '.join(part) origin = '' t = exchange.get('0', '') if t.lower() == data['word'].lower(): del exchange['0'] if '1' in exchange: del exchange['1'] if '0' in exchange: t = exchange['0'] if t != data['word']: origin = t derive = '' if '1' in exchange: t = exchange['1'] p = [] if 'p' in t and 'd' in t: derive = u'过去式和过去分词' elif 's' in t and '3' in t: derive = u'第三人称单数' else: for x in ('i', 'p', 'd', '3', 's', 'r', 't'): if x in t: derive = stardict.tools._exchanges[x] break if derive: origin = data['word'] + u' 是 ' + origin + u' 的' + derive better = '' if ('r' in exchange) and ('t' in exchange): better = exchange['r'] + ', ' + exchange['t'] lines = [] # if text and (not exchange.get('1', '') in ('p', 'd', 'i', '3', 'pd', 'dp')): if text: if style == 0: lines.append(u'[时态] ' + text) else: lines.append(u'时态: ' + text) if better and (not exchange.get('1', '') in ('r', 't')): if style == 0: lines.append(u'[级别] ' + better) else: lines.append(u'级别: ' + better) if origin: if style == 0: lines.append(u'[原型] ' + origin) else: lines.append(u'原型: ' + origin) return '\n'.join(lines) def word_pos (self, data): pos = stardict.tools.pos_extract(data) if not pos: return None if len(pos) < 2: return None text = [] for mode, num in pos: text.append('%s(%s%%)'%(mode[0], num)) desc = ', '.join(text) return desc.replace('\\', '').replace('\n', '') def text2html (self, text): import cgi return cgi.escape(text, True).replace('\n', '
') # 导出星际译王的词典源文件，用于 DictEditor 转换 def compile_stardict (self, dictionary, filename, title): print('generating ...') words = stardict.tools.dump_map(dictionary, False) out = {} pc = stardict.tools.progress(len(words)) for word in words: pc.next() data = dictionary[word] phonetic = data['phonetic'] translation = data['translation'] if not translation: translation = data['definition'] if not translation: print('missing: %s'%word) continue head = self.word_level(data) tag = self.word_tag(data) if phonetic: if head: text = '*[' + phonetic + '] -' + head + '\n' else: text = '*[' + phonetic + ']\n' elif head: text = '-' + head + '\n' else: text = '' text = text + translation exchange = self.word_exchange(data, 0) if exchange: text = text + '\n\n' + exchange + '' if tag: text = text + '\n' + '(' + tag + ')' out[word] = text pc.done() print('saving ...') stardict.tools.export_stardict(out, filename, title) return pc.count # 导出 Mdx 源文件，然后可以用 MdxBuilder 转换成 .mdx词典 def compile_mdx (self, dictionary, filename, mode = None, style = False): words = stardict.tools.dump_map(dictionary, False) fp = codecs.open(filename, 'w', 'utf-8') text2html = self.text2html pc = stardict.tools.progress(len(words)) if mode is None: mode = ('name', 'phonetic') count = 0 stripword = stardict.stripword words = [ k for k in words ] words.sort(key = lambda x: stripword(x)) for word in words: pc.next() data = dictionary[word] phonetic = data['phonetic'] translation = data['translation'] if not translation: translation = data['definition'] if not translation: continue # if pc.count >= 100000: # break head = self.word_level(data) tag = self.word_tag(data) fp.write(word.replace('\r', '').replace('\n', '') + '\r\n') if 'name' in mode: if not style: fp.write('%s'%text2html(word)) fp.write('

\r\n') else: fp.write('`1`%s`2``2`\r\n'%text2html(word)) if 'phonetic' in mode: if phonetic or head: if phonetic: if not style: fp.write('') fp.write(text2html(u'[%s]'%phonetic)) fp.write('') else: fp.write('`3`' + text2html(u'[%s]'%phonetic)) if head: if phonetic: fp.write(' ') if not style: fp.write('') fp.write(text2html(u'-%s'%head)) fp.write('') else: fp.write('`4`' + text2html(u'-%s'%head)) if not style: fp.write('

\r\n') else: fp.write('`2``2`\r\n') for line in translation.split('\n'): line = line.rstrip('\r\n ') fp.write(text2html(line) + '
\r\n') if (not 'phonetic' in mode) and head: if tag: tag = tag + ' -' + head else: tag = '-' + head exchange = self.word_exchange(data, 1) if exchange: if not style: fp.write('
') fp.write(text2html(exchange)) fp.write('\r\n') else: fp.write(u'`2``4`' + text2html(exchange) + '`2`\r\n') if tag: if not style: fp.write('
') fp.write('(%s)'%text2html(tag)) fp.write('\r\n') else: fp.write('`2``4`(%s)\r\n'%text2html(tag)) fp.write('') if count < len(words) - 1: fp.write('\r\n') count += 1 pc.done() return pc.count def _split_pos (self, text): pos = text.find('.') if pos < 0: return '', text if text[:pos].isalpha() and pos < 8: return text[:pos + 1], text[pos+1:].lstrip('\t ') return '', text # 生成支持 css 的 tag def _generate_tag (self, fp, data): tag = data.get('tag') frq = data.get('frq') bnc = data.get('bnc') if (not tag) and (not frq) and (not bnc): return False text2html = self.text2html out = fp.write outline = lambda x: fp.write(x + '\r\n') outtext = lambda x: fp.write(text2html(x)) return True # 生成支持 css 的 html def _generate_html (self, fp, data): text2html = self.text2html out = fp.write outline = lambda x: fp.write(x + '\r\n') outtext = lambda x: fp.write(text2html(x)) word = data['word'] phonetic = data['phonetic'] translation = data['translation'] if not translation: translation = data['definition'] if not translation: return False outline('
') outline('
') # word head outline('
%s
'%text2html(word)) outline('
') # phonetic and tag head = self.word_level(data) if phonetic or head: outline('
') if phonetic: outline(' [%s]'%text2html(phonetic)) if head: outline(' -') if data.get('oxford'): t = u'Oxford 3000 Keywords' p = u'\u203B' outline(' %s'%(t, p)) collins = data.get('collins', '0') if isinstance(collins, str) or isinstance(collins, unicode): if collins in ('', '0'): collins = 0 else: collins = int(collins) if collins: title = 'Collins Stars' out(' '%title) out(u'\u2605' * int(collins)) outline('') outline('
') # translation outline('
') for line in translation.split('\n'): line = line.rstrip('\r\n') outline('
') if line[:4] == u'[网络]': text = text2html(line[4:].lstrip('\t ')) outline(u' [网络]') outline(u' %s'%text) elif line[:1] == '>': text = text2html(line) outline(u' %s'%text) else: pos, text = self._split_pos(line) if pos: outline(' %s'%text2html(pos)) if text: outline(' %s'%text2html(text)) outline('
') outline('
') # exchange exchange = self.word_exchange(data, 0) if exchange: outline('
') for line in exchange.split('\n'): line = line.rstrip('\r\n\t ') if line.startswith(u'[时态]'): text = text2html(line[4:].lstrip(' ')) outline('
') outline(' %s'%u'时态:') outline(' %s'%text) outline('
') elif line.startswith(u'[级别]'): text = text2html(line[4:].lstrip(' ')) outline('
') outline(' %s'%u'级别:') outline(' %s'%text) outline('
') elif line.startswith(u'[原型]'): text = text2html(line[4:].lstrip(' ')) outline('
') outline(' %s'%u'原型:') outline(' %s'%text) outline('
') outline('
') # tag tag = self.word_tag(data) if tag: title = '' frq = data.get('frq') bnc = data.get('bnc') if frq: title = u'COCA: %s'%frq if bnc: if title: title += ', ' title += 'BNC: %s'%bnc outline('
'%title) outline(' (' + text2html(tag) + ')') outline('
') # finalize outline('
') outline('
') outline('
') return True def compile_css (self, dictionary, filename, css = None): fp = codecs.open(filename, 'w', 'utf-8') text2html = self.text2html pc = stardict.tools.progress(len(dictionary)) if not css: main = os.path.split(filename)[-1] css = os.path.splitext(main)[0] + '.css' for _, word in dictionary: pc.next() data = dictionary.query(word) translation = data['translation'] if not translation: translation = data['definition'] if not translation: continue fp.write(word.replace('\r', '').replace('\n', '') + '\r\n') fp.write('\r\n'%css) self._generate_html(fp, data) fp.write('') if pc.count < pc.total: fp.write('\r\n') fp.close() pc.done() return 0 def list_load (self, filename, encoding = 'utf-8'): words = {} import codecs with codecs.open(filename, encoding = encoding) as fp: for line in fp: line = line.strip('\r\n\t ') if not line: continue words[line] = 1 return words def list_save (self, filename, words): import codecs with codecs.open(filename, 'w', encoding = 'utf-8') as fp: for w in words: fp.write(w + '\n') return True def mdict2eudic (self, mdx_src, outname, skip = True): import codecs with codecs.open(mdx_src, encoding = 'utf-8') as srcfp: fp = codecs.open(outname, 'w', encoding = 'utf-8') word = None part = [] count = 0 valid = 0 for line in srcfp: line = line.strip('\r\n\t ') if not line: continue if word is None: word = line part = [] elif line != '': part.append(line) else: invalid = False if skip: for ch in word: if ord(ch) >= 128: invalid = True break if not invalid: text = ''.join(part) if (not word[:1] == '-') and (not word[-1:] == '-'): fp.write(word + '@' + text + '\r\n') valid += 1 word = None part = [] count += 1 if count % 10000 == 0: print('current count=%d'%count) print('done valid=%d/%d'%(valid, count)) return True def load_index (self, filename, encoding = 'utf-8', lower = False): words = {} for line in codecs.open(filename, encoding = encoding): line = line.strip('\r\n\t ') if not line: continue if lower: words[line.lower()] = line else: words[line] = line return words #---------------------------------------------------------------------- # 解析 resemble.txt 生成辨析释义 #---------------------------------------------------------------------- class Resemble (object): def __init__ (self): self._resembles = [] self._words = {} self._filename = None self._lineno = 0 def error (self, text): t = '%s:%s: error: %s\n' t = t%(self._filename, self._lineno, text) sys.stderr.write(t) sys.stderr.flush() def load (self, filename): self._resembles = [] self._words = {} file_content = stardict.tools.load_text(filename) if file_content is None: sys.stderr.write('cannot read: %s\n'%filename) return False key = None content = [] self._filename = filename self._lineno = 0 for line in file_content.split('\n'): line = line.strip('\r\n\t ') self._lineno += 1 if key is None: if not line: continue if line[:1] != '%': self.error('must starts with a percent sign') return False line = line[1:].lstrip('\r\n\t ') key = [ n.strip('\r\n\t ') for n in line.split(',') ] if not key: self.error('empty heading words') return False for word in key: if not word: self.error('empty item') return False content = [] else: if not line: wt = {} uuid = [ n for n in key ] uuid.sort() wt['words'] = tuple(key) wt['content'] = content wt['uuid'] = ', '.join(uuid) self._resembles.append(wt) key = None content = [] elif line[:1] == '-': line = line[1:].lstrip('\r\n\t') pos = line.find(':') if pos < 0: self.error('expect colon') word = line[:pos].strip('\r\n\t ') text = line[pos+1:].strip('\r\n\t ') text = text.replace('\\n', '\n') content.append((word, text)) else: content.append(line) if key: wt = {'words':tuple(key), 'content':content} uuid = [ n for n in key ] uuid.sort() wt['uuid'] = uuid self._resembles.append(wt) self._init_refs() return True def _init_refs (self): self._words = {} words = {} existence = {} for wt in self._resembles: for word in wt['words']: if not word in words: words[word] = [] if not word in existence: existence[word] = {} uuid = wt['uuid'] if uuid in existence[word]: continue words[word].append(wt) existence[word][uuid] = 1 for word in words: self._words[word] = tuple(words[word]) return True def __len__ (self): return len(self._resembles) def __getitem__ (self, key): if isinstance(key, int) or isinstance(key, long): return self._resembles[key] return self._words[key] def __contains__ (self, key): if isinstance(key, int) or isinstance(key, long): if key < 0 or key >= len(self._resembles): return False elif not key in self._words: return False return True def __iter__ (self): return self._resembles.__iter__() def text2html (self, text): import cgi return cgi.escape(text, True).replace('\n', '
') def dump_text (self, wt): lines = [] lines.append('% ' + (', '.join(wt['words']))) for content in wt['content']: if isinstance(content, list) or isinstance(content, tuple): word, text = content text = text.replace('\n', '\\n') lines.append('- ' + word + ': ' + text) else: lines.append(content) return '\n'.join(lines) def dump_html (self, wt, style = 0): lines = [] text2html = self.text2html lines.append('
') text = ', '.join(wt['words']) text = '
' + text2html(text) + '
' lines.append(text) lines.append('
') for content in wt['content']: if isinstance(content, tuple) or isinstance(content, list): head = content[0] desc = [ n.rstrip('\n') for n in content[1].split('\n') ] text = '%s: '%text2html(head) text = text + text2html(desc[0]) lines.append(text + '
') for line in desc[1:]: line = line.strip('\r\n\t ') if not line: continue if style == 0: lines.append(text2html(line) + '
') elif style == 1: pos = -1 for i in xrange(len(line)): if ord(line[i]) >= 128: pos = i break if pos < 0: en, cn = line, '' else: en, cn = line[:pos], line[pos:] en = text2html(en.strip('\r\n\t ')) cn = text2html(cn.strip('\r\n\t ')) line = u' • ' if en: line += '%s  '%en if cn: line += ' %s'%cn lines.append('' + line + '
') else: lines.append(text2html(content) + '
') lines.append('
') lines.append('
') return '\n'.join(lines) def compile_map (self): words = {} if (not self._resembles) or (not self._words): return False pc = stardict.tools.progress(len(self._words)) for word in self._words: pc.next() if not word: continue wts = [ self.dump_html(wt, 1) for wt in self._words[word] ] words[word] = '
\n'.join(wts) return words def compile_mdx (self, filename): words = self.compile_map() title = u'有道词语辨析' text = time.strftime('%Y-%m-%d %H:%M:%S') desc = u'\n' desc += u'有道词语辨析
\n' desc += u'词条数：%d
\n'%len(self._words) desc += u'词组数：%d
\n'%len(self._resembles) desc += u'作者：skywind
\n' desc += u'日期：%s
\n'%text desc += '' stardict.tools.export_mdx(words, filename, title, desc) return True #---------------------------------------------------------------------- # Treasure #---------------------------------------------------------------------- class Treasure (object): def __init__ (self): self.mark1 = '' self.generator = Generator() def text2html (self, text): import cgi return cgi.escape(text, True).replace('\n', '
') def clear_html (self, text): return text.replace('<', '').replace('>', '').replace('&', '') def detail (self, data, name, default = None): detail = data.get('detail') if not detail: return default return detail.get(name, default) def define_html (self, definition, plain = False): lines = [] if plain: return self.text2html(definition) text2html = self.text2html for line in definition.split('\n'): line = line.rstrip('\r\n\t ') if not line: pass pos = line.find('.') head = '' if pos > 0 and line[:pos].strip('\t ').isalpha(): if pos < 8: head = line[:pos+1].rstrip(' ') line = line[pos+1:].lstrip(' ') text = '' if head: text += self.mark1 text += text2html(head) text += ' ' text += text2html(line) lines.append(text) return '
\n'.join(lines) def get_definition (self, data, plain = False): definition = data['definition'] if not definition: return None return self.define_html(definition, plain) def get_translation (self, data, plain = False): translation = data['translation'] if not translation: return None return self.define_html(translation, plain) def get_phonetic (self, data): phonetic = data['phonetic'] if not phonetic: return None return '[' + self.clear_html(phonetic) + ']' def get_level (self, data): text = self.generator.word_tag(data) head = self.generator.word_level(data) if head: if text: text += ' -' + head else: text = '-' + head if text: return self.clear_html(u'(%s)'%text) return None def get_exchange (self, data): text = '' exchange = data.get('exchange') if not exchange: return None chg = stardict.tools.exchange_loads(exchange) if not chg: return None part = [] last = '' count = 0 for k in ('p', 'd', 'i', '3'): p = chg.get(k) if p: count += 1 if p != last: part.append(u'%s'%p) last = p if count == 4: text = ', '.join(part) return self.clear_html(u'时态：' + text) if ('r' in chg) and ('t' in chg): text = ', '.join([chg['r'], chg['t']]) return self.clear_html(u'级别：' + text) return None def get_syno (self, data, plain = False): detail = data.get('detail') if not detail: return None syno = detail.get('syno') if not syno: return None lines = [] for group in syno: text = group[0] word = ', '.join(group[1]) lines.append('' + self.define_html(text, plain) + '') text = ' - ' + self.text2html(word) + '' lines.append(text) return '
\n'.join(lines) def get_proportion (self, data): detail = data.get('detail') if not detail: return None return detail.get('proportion') def get_cald (self, data): detail = data.get('detail') if not detail: return None html = detail.get('cald') if not html: return None text = html mark = '
' p1 = text.find(mark) if p1 >= 0: text = text[p1 + len(mark):] test = '[' p1 = text.find(test) if p1 >= 0: p1 = text.find(']', p1) if p1 >= 0: text = text[p1+8:] newmark = '
' text = text.strip('\n\r ') text = text.replace(mark, newmark + '\n') return text def get_collins (self, data): return self.detail(data, 'collins', None) def get_memo (self, data): detail = data.get('detail') output = [] if not detail: detail = {} youci = detail.get('youci') if youci: p1 = youci.find('
\n') if p1 >= 0: youci = youci[p1 + 5:] if youci: head = u'【优词】　 ' head = '' output.append(head + youci) xdf = detail.get('xdf') if xdf: head = u'【新东方】　' head = '' output.append(head + xdf) bzsd = detail.get('bzsd') if bzsd: head = u'【不择手段】　' head = '' output.append(head + self.text2html(bzsd)) if not output: return None return '

\n'.join(output) def get_extra (self, data): detail = data.get('detail') if not detail: return None output = [] resemble = detail.get('resemble') if resemble: head = u'【有道词语辨析】
\n' head = '' output.append(head + resemble) syno = detail.get('syno') if syno: head = u'【有道近义词】
\n' head = '' output.append(head + self.get_syno(data)) if not output: return None return '
\n'.join(output) def get_explain (self, data): cald = self.get_cald(data) if cald: return cald return self.get_collins(data) def generate_front (self, data): html = [] text = "

%s

" html.append(text%self.text2html(data['word'])) html.append("
") text = "%s" html.append(text%self.get_phonetic(data)) text = "%s" html.append(text%self.get_level(data)) html.append('
') return '\n'.join(html) def generate_back (self, data): html = [] html.append('
') hr = "height:1px;border:none;border-top:1px dashed #0066CC;" hr = hr + "background-color:#ffffff;" hr = '
'%hr text = "
%s
" html.append(text%self.get_translation(data)) html.append('
') exchange = self.get_exchange(data) if exchange: text = "
%s
" html.append(text%exchange) proportion = self.get_proportion(data) if proportion: text = u"
分布：%s
" html.append(text%proportion) html.append(hr) memo = self.get_memo(data) if memo: html.append('
') html.append(memo) html.append('
') html.append(hr) explain = self.get_explain(data) if explain: html.append('
') html.append(explain) html.append('
') extra = self.get_extra(data) if extra: html.append(hr) html.append('
') html.append(extra) html.append('
') html.append('
') return '\n'.join(html) def compile_mdx (self, db, name1, name2): mdx1 = {} mdx2 = {} pc = stardict.tools.progress(len(db)) for _, word in db: pc.next() data = db[word] mdx1[word] = self.generate_front(data) mdx2[word] = self.generate_back(data) pc.done() if os.path.splitext(name1)[-1].lower() == '.mdx': stardict.tools.export_mdx(mdx1, name1, 'anki-front') else: stardict.tools.export_mdict(mdx1, name1) if os.path.splitext(name2)[-1].lower() == '.mdx': stardict.tools.export_mdx(mdx2, name2, 'anki-back') else: stardict.tools.export_mdict(mdx2, name2) return 0 #---------------------------------------------------------------------- # generation #---------------------------------------------------------------------- generator = Generator() resemble = Resemble() treasure = Treasure() #---------------------------------------------------------------------- # testing case #---------------------------------------------------------------------- if __name__ == '__main__': def test1(): db = stardict.open_local('stardict.db') data = db['higher'] # data = {'exchange':'p:P/d:D/i:I/0:haha'} print(generator.word_exchange(data, 0)) print(generator.word_exchange(data, 1)) def test2(): resemble.load('resemble.txt') # print resemble.dump_text(resemble[0]) for wt in resemble['stimulate']: print resemble.dump_html(wt, 1).encode('gbk', 'ignore') print '' return 0 def test3(): if not resemble.load('resemble.txt'): return -1 fn = u'd:/Program Files/GoldenDict/content/others/有道词语辨析.mdx' resemble.compile_mdx(fn) return 0 def test4(): db = stardict.open_local('treasure.db') data = db['breakup'] # html = treasure.define_html(data['translation']) html = treasure.get_collins(data).encode('gbk', 'ignore') print html def test5(): name1 = 'anki-front.txt' name2 = 'anki-back.txt' home = 'd:/Program Files/GoldenDict/content/Others/' home = '../../../work/' db = stardict.open_local('treasure.db') treasure.compile_mdx(db, home + name1, home + name2) def test6(): db = stardict.open_local('ultimate.db') data = db['sting'] data['translation'] += u'\n> hahahah\n[网络] 你好' import StringIO sio = StringIO.StringIO() generator._generate_html(sio, data) print sio.getvalue().encode('gbk', 'ignore') test6()