321 lines
7.9 KiB
Python
321 lines
7.9 KiB
Python
#! /usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
#======================================================================
|
|
#
|
|
# linguist.py -
|
|
#
|
|
# Created by skywind on 2017/03/22
|
|
# Last change: 2017/03/22 13:44:42
|
|
#
|
|
#======================================================================
|
|
import sys, os, time
|
|
|
|
# https://www.nodebox.net/code/index.php/Linguistics
|
|
|
|
#----------------------------------------------------------------------
|
|
# python 2/3 compatible
|
|
#----------------------------------------------------------------------
|
|
if sys.version_info[0] >= 3:
|
|
long = int
|
|
xrange = range
|
|
unicode = str
|
|
|
|
|
|
#----------------------------------------------------------------------
|
|
# 词形变换
|
|
#----------------------------------------------------------------------
|
|
class WordHelper (object):
|
|
|
|
# 初始化
|
|
def __init__ (self):
|
|
self.__lemmatizer = None
|
|
|
|
# 取得 WordNet 的词定义
|
|
def definition (self, word, txt = False):
|
|
from nltk.corpus import wordnet as wn
|
|
syns = wn.synsets(word)
|
|
output = []
|
|
for syn in syns:
|
|
name = syn.name()
|
|
part = name.split('.')
|
|
mode = part[1]
|
|
output.append((mode, syn.definition()))
|
|
if txt:
|
|
output = '\n'.join([ (m + ' ' + n) for m, n in output ])
|
|
return output
|
|
|
|
# 取得动词的：-ing, -ed, -en, -s
|
|
# NodeBox 的 Linguistics 软件包 11487 个动词只能处理 6722 个
|
|
def verb_tenses (self, word):
|
|
word = word.lower()
|
|
if ' ' in word:
|
|
return None
|
|
import en
|
|
if not en.is_verb(word):
|
|
return None
|
|
tenses = {}
|
|
try:
|
|
tenses['i'] = en.verb.present_participle(word)
|
|
tenses['p'] = en.verb.past(word)
|
|
tenses['d'] = en.verb.past_participle(word)
|
|
tenses['3'] = en.verb.present(word, person = 3, negate = False)
|
|
except:
|
|
return None
|
|
valid = True
|
|
for k in tenses:
|
|
v = tenses[k]
|
|
if not v:
|
|
valid = False
|
|
break
|
|
elif "'" in v:
|
|
valid = False
|
|
break
|
|
if not valid:
|
|
return None
|
|
return tenses
|
|
|
|
# 名词的复数：有时候不可数名词也会被加上 -s，需要先判断是否可数（语料库）
|
|
def noun_plural (self, word, method = 0):
|
|
plural = None
|
|
if method == 0:
|
|
import en
|
|
plural = en.noun.plural(word)
|
|
elif method == 1:
|
|
import pattern.en
|
|
plural = pattern.en.pluralize(word)
|
|
elif method == 2:
|
|
import inflect
|
|
plural = inflect.pluralize(word)
|
|
elif method < 0:
|
|
plural = self.noun_plural(word, 0)
|
|
if not plural:
|
|
plural = self.noun_plural(word, 1)
|
|
if not plural:
|
|
try:
|
|
import inflect
|
|
plural = inflect.pluralize(word)
|
|
except:
|
|
pass
|
|
if not plural:
|
|
return None
|
|
return plural
|
|
|
|
# 求解比较级
|
|
def adjective_comparative (self, word):
|
|
import pattern.en
|
|
return pattern.en.comparative(word)
|
|
|
|
# 求解最高级
|
|
def adjective_superlative (self, word):
|
|
import pattern.en
|
|
return pattern.en.superlative(word)
|
|
|
|
# 求解复数，使用 pattern.en 软件包
|
|
def pluralize (self, word):
|
|
import pattern.en
|
|
return pattern.en.pluralize(word)
|
|
|
|
# 取得所有动词
|
|
def all_verbs (self):
|
|
import en
|
|
words = []
|
|
for n in en.wordnet.all_verbs():
|
|
words.append(n.form)
|
|
return words
|
|
|
|
# 取得所有副词
|
|
def all_adverbs (self):
|
|
import en
|
|
words = []
|
|
for n in en.wordnet.all_adverbs():
|
|
words.append(n.form)
|
|
return words
|
|
|
|
# 取得所有形容词
|
|
def all_adjectives (self):
|
|
import en
|
|
words = []
|
|
for n in en.wordnet.all_adjectives():
|
|
words.append(n.form)
|
|
return words
|
|
|
|
# 取得所有名词
|
|
def all_nouns (self):
|
|
import en
|
|
words = []
|
|
for n in en.wordnet.all_nouns():
|
|
words.append(n.form)
|
|
return words
|
|
|
|
# 返回原始单词
|
|
def lemmatize (self, word, pos = 'n'):
|
|
word = word.lower()
|
|
if self.__lemmatizer is None:
|
|
from nltk.stem.wordnet import WordNetLemmatizer
|
|
self.__lemmatizer = WordNetLemmatizer()
|
|
return self.__lemmatizer.lemmatize(word, pos)
|
|
|
|
|
|
#----------------------------------------------------------------------
|
|
# global
|
|
#----------------------------------------------------------------------
|
|
tools = WordHelper()
|
|
|
|
#----------------------------------------------------------------------
|
|
# WordRoot
|
|
#----------------------------------------------------------------------
|
|
class WordRoot (object):
|
|
|
|
def __init__ (self, root):
|
|
self.root = root
|
|
self.count = 0
|
|
self.words = {}
|
|
|
|
def add (self, c5, word, n = 1):
|
|
if c5 and word:
|
|
term = (c5, word)
|
|
if not term in self.words:
|
|
self.words[term] = n
|
|
else:
|
|
self.words[term] += n
|
|
self.count += n
|
|
return True
|
|
|
|
def dump (self):
|
|
output = []
|
|
for term in self.words:
|
|
c5, word = term
|
|
output.append((c5, word, self.words[term]))
|
|
output.sort(key = lambda x: (x[2], x[0]), reverse = True)
|
|
return output
|
|
|
|
def __len__ (self):
|
|
return len(self.words)
|
|
|
|
def __getitem__ (self, key):
|
|
return self.words[key]
|
|
|
|
|
|
|
|
#----------------------------------------------------------------------
|
|
# testing
|
|
#----------------------------------------------------------------------
|
|
if __name__ == '__main__':
|
|
def test1():
|
|
for word in ['was', 'gave', 'be', 'bound']:
|
|
print('%s -> %s'%(word, tools.lemmatize(word, 'v')))
|
|
return 0
|
|
def test2():
|
|
import ascmini
|
|
rows = ascmini.csv_load('bnc-words.csv')
|
|
output = []
|
|
words = {}
|
|
for row in rows:
|
|
root = row[0]
|
|
size = int(row[1])
|
|
c5 = row[2]
|
|
word = row[3]
|
|
count = int(row[4])
|
|
head = root[:1].lower()
|
|
if size <= 1:
|
|
continue
|
|
if count * 1000 / size < 1:
|
|
continue
|
|
if '*' in word:
|
|
continue
|
|
if c5 in ('UNC', 'CRD'):
|
|
continue
|
|
if '(' in root or '/' in root:
|
|
continue
|
|
if head != '\'' and (not head.isalpha()):
|
|
if head.isdigit():
|
|
continue
|
|
if head in ('$', '#', '-'):
|
|
continue
|
|
if root.count('\'') >= 2:
|
|
continue
|
|
if not root in words:
|
|
stem = WordRoot(root)
|
|
words[root] = stem
|
|
else:
|
|
stem = words[root]
|
|
stem.add(c5, word.lower(), count)
|
|
for key in words:
|
|
stem = words[key]
|
|
for c5, word, count in stem.dump():
|
|
output.append((stem.root, stem.count, c5, word, count))
|
|
output.sort(key = lambda x: (x[1], x[0]), reverse = True)
|
|
# ascmini.csv_save(output, 'bnc-clear.csv')
|
|
print 'count', len(words)
|
|
def test3():
|
|
import ascmini
|
|
rows = ascmini.csv_load('bnc-clear.csv')
|
|
output = []
|
|
words = {}
|
|
for row in rows:
|
|
root = row[0]
|
|
size = int(row[1])
|
|
c5 = row[2]
|
|
word = row[3].lower()
|
|
count = int(row[4])
|
|
if word == root:
|
|
continue
|
|
if not root in words:
|
|
stem = WordRoot(root)
|
|
words[root] = stem
|
|
else:
|
|
stem = words[root]
|
|
stem.add('*', word, count)
|
|
stem.count = size
|
|
fp = open('bnc-lemma.txt', 'w')
|
|
lemmas = []
|
|
for key in words:
|
|
stem = words[key]
|
|
part = []
|
|
for c5, word, count in stem.dump():
|
|
output.append((stem.root, stem.count, c5, word, count))
|
|
part.append('%s/%d'%(word, count))
|
|
if not part:
|
|
continue
|
|
text = '%s/%d -> '%(stem.root, stem.count)
|
|
lemmas.append((stem.count, stem.root, text + ','.join(part)))
|
|
output.sort(key = lambda x: (x[1], x[0]), reverse = True)
|
|
lemmas.sort(reverse = True)
|
|
for _, _, text in lemmas:
|
|
fp.write(text + '\n')
|
|
ascmini.csv_save(output, 'bnc-test.csv')
|
|
print 'count', len(words)
|
|
return 0
|
|
def test4():
|
|
import stardict
|
|
lm1 = stardict.LemmaDB()
|
|
lm2 = stardict.LemmaDB()
|
|
lm1.load('bnc-lemma.txt')
|
|
lm2.load('lemma.en.txt')
|
|
count1 = 0
|
|
count2 = 0
|
|
for stem in lm2.dump('stem'):
|
|
childs = lm2.get(stem)
|
|
stem = stem.lower()
|
|
if len(stem) <= 2 and stem.isupper():
|
|
continue
|
|
if not stem in lm1:
|
|
count1 += 1
|
|
else:
|
|
obj = lm1.get(stem)
|
|
for word in childs:
|
|
word = word.lower()
|
|
if not word in obj:
|
|
print '%s -> %s'%(stem, word)
|
|
count2 += 1
|
|
for word in childs:
|
|
lm1.add(stem, word.lower())
|
|
print 'count', count1, count2
|
|
lm1.save('lemma-bnc.txt')
|
|
return 0
|
|
test4()
|
|
|
|
|
|
|
|
|