remove old commits

linguist.py

@@ -0,0 +1,320 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#======================================================================
+#
+# linguist.py - 
+#
+# Created by skywind on 2017/03/22
+# Last change: 2017/03/22 13:44:42
+#
+#======================================================================
+import sys, os, time
+
+# https://www.nodebox.net/code/index.php/Linguistics
+
+#----------------------------------------------------------------------
+# python 2/3 compatible
+#----------------------------------------------------------------------
+if sys.version_info[0] >= 3:
+	long = int
+	xrange = range
+	unicode = str
+
+
+#----------------------------------------------------------------------
+# 词形变换
+#----------------------------------------------------------------------
+class WordHelper (object):
+
+	# 初始化
+	def __init__ (self):
+		self.__lemmatizer = None
+
+	# 取得 WordNet 的词定义
+	def definition (self, word, txt = False):
+		from nltk.corpus import wordnet as wn
+		syns = wn.synsets(word)
+		output = []
+		for syn in syns:
+			name = syn.name()
+			part = name.split('.')
+			mode = part[1]
+			output.append((mode, syn.definition()))
+		if txt:
+			output = '\n'.join([ (m + ' ' + n) for m, n in output ])
+		return output
+
+	# 取得动词的：-ing, -ed, -en, -s
+	# NodeBox 的 Linguistics 软件包 11487 个动词只能处理 6722 个 
+	def verb_tenses (self, word):
+		word = word.lower()
+		if ' ' in word:
+			return None
+		import en
+		if not en.is_verb(word):
+			return None
+		tenses = {}
+		try:
+			tenses['i'] = en.verb.present_participle(word)
+			tenses['p'] = en.verb.past(word)
+			tenses['d'] = en.verb.past_participle(word)
+			tenses['3'] = en.verb.present(word, person = 3, negate = False)
+		except:
+			return None
+		valid = True
+		for k in tenses:
+			v = tenses[k]
+			if not v:
+				valid = False
+				break
+			elif "'" in v:
+				valid = False
+				break
+		if not valid:
+			return None
+		return tenses
+
+	# 名词的复数：有时候不可数名词也会被加上 -s，需要先判断是否可数（语料库）
+	def noun_plural (self, word, method = 0):
+		plural = None
+		if method == 0:
+			import en
+			plural = en.noun.plural(word)
+		elif method == 1:
+			import pattern.en
+			plural = pattern.en.pluralize(word)
+		elif method == 2:
+			import inflect
+			plural = inflect.pluralize(word)
+		elif method < 0:
+			plural = self.noun_plural(word, 0)
+			if not plural:
+				plural = self.noun_plural(word, 1)
+				if not plural:
+					try:
+						import inflect
+						plural = inflect.pluralize(word)
+					except:
+						pass
+		if not plural:
+			return None
+		return plural
+
+	# 求解比较级
+	def adjective_comparative (self, word):
+		import pattern.en
+		return pattern.en.comparative(word)
+
+	# 求解最高级
+	def adjective_superlative (self, word):
+		import pattern.en
+		return pattern.en.superlative(word)
+
+	# 求解复数，使用 pattern.en 软件包
+	def pluralize (self, word):
+		import pattern.en
+		return pattern.en.pluralize(word)
+
+	# 取得所有动词
+	def all_verbs (self):
+		import en
+		words = []
+		for n in en.wordnet.all_verbs():
+			words.append(n.form)
+		return words
+
+	# 取得所有副词
+	def all_adverbs (self):
+		import en
+		words = []
+		for n in en.wordnet.all_adverbs():
+			words.append(n.form)
+		return words
+
+	# 取得所有形容词
+	def all_adjectives (self):
+		import en
+		words = []
+		for n in en.wordnet.all_adjectives():
+			words.append(n.form)
+		return words
+
+	# 取得所有名词
+	def all_nouns (self):
+		import en
+		words = []
+		for n in en.wordnet.all_nouns():
+			words.append(n.form)
+		return words
+
+	# 返回原始单词
+	def lemmatize (self, word, pos = 'n'):
+		word = word.lower()
+		if self.__lemmatizer is None:
+			from nltk.stem.wordnet import WordNetLemmatizer
+			self.__lemmatizer = WordNetLemmatizer()
+		return self.__lemmatizer.lemmatize(word, pos)
+
+
+#----------------------------------------------------------------------
+# global
+#----------------------------------------------------------------------
+tools = WordHelper()
+
+#----------------------------------------------------------------------
+# WordRoot
+#----------------------------------------------------------------------
+class WordRoot (object):
+
+	def __init__ (self, root):
+		self.root = root
+		self.count = 0
+		self.words = {}
+
+	def add (self, c5, word, n = 1):
+		if c5 and word:
+			term = (c5, word)
+			if not term in self.words:
+				self.words[term] = n
+			else:
+				self.words[term] += n
+			self.count += n
+		return True
+
+	def dump (self):
+		output = []
+		for term in self.words:
+			c5, word = term
+			output.append((c5, word, self.words[term]))
+		output.sort(key = lambda x: (x[2], x[0]), reverse = True)
+		return output
+
+	def __len__ (self):
+		return len(self.words)
+	
+	def __getitem__ (self, key):
+		return self.words[key]
+
+
+
+#----------------------------------------------------------------------
+# testing
+#----------------------------------------------------------------------
+if __name__ == '__main__':
+	def test1():
+		for word in ['was', 'gave', 'be', 'bound']:
+			print('%s -> %s'%(word, tools.lemmatize(word, 'v')))
+		return 0
+	def test2():
+		import ascmini
+		rows = ascmini.csv_load('bnc-words.csv')
+		output = []
+		words = {}
+		for row in rows:
+			root = row[0]
+			size = int(row[1])
+			c5 = row[2]
+			word = row[3]
+			count = int(row[4])
+			head = root[:1].lower()
+			if size <= 1:
+				continue
+			if count * 1000 / size < 1:
+				continue
+			if '*' in word:
+				continue
+			if c5 in ('UNC', 'CRD'):
+				continue
+			if '(' in root or '/' in root:
+				continue
+			if head != '\'' and (not head.isalpha()):
+				if head.isdigit():
+					continue
+				if head in ('$', '#', '-'):
+					continue
+			if root.count('\'') >= 2:
+				continue
+			if not root in words:
+				stem = WordRoot(root)
+				words[root] = stem
+			else:
+				stem = words[root]
+			stem.add(c5, word.lower(), count)
+		for key in words:
+			stem = words[key]
+			for c5, word, count in stem.dump():
+				output.append((stem.root, stem.count, c5, word, count))
+		output.sort(key = lambda x: (x[1], x[0]), reverse = True)
+		# ascmini.csv_save(output, 'bnc-clear.csv')
+		print 'count', len(words)
+	def test3():
+		import ascmini
+		rows = ascmini.csv_load('bnc-clear.csv')
+		output = []
+		words = {}
+		for row in rows:
+			root = row[0]
+			size = int(row[1])
+			c5 = row[2]
+			word = row[3].lower()
+			count = int(row[4])
+			if word == root:
+				continue
+			if not root in words:
+				stem = WordRoot(root)
+				words[root] = stem
+			else:
+				stem = words[root]
+			stem.add('*', word, count)
+			stem.count = size
+		fp = open('bnc-lemma.txt', 'w')
+		lemmas = []
+		for key in words:
+			stem = words[key]
+			part = []
+			for c5, word, count in stem.dump():
+				output.append((stem.root, stem.count, c5, word, count))
+				part.append('%s/%d'%(word, count))
+			if not part:
+				continue
+			text = '%s/%d -> '%(stem.root, stem.count)
+			lemmas.append((stem.count, stem.root, text + ','.join(part)))
+		output.sort(key = lambda x: (x[1], x[0]), reverse = True)
+		lemmas.sort(reverse = True)
+		for _, _, text in lemmas:
+			fp.write(text + '\n')
+		ascmini.csv_save(output, 'bnc-test.csv')
+		print 'count', len(words)
+		return 0
+	def test4():
+		import stardict
+		lm1 = stardict.LemmaDB()
+		lm2 = stardict.LemmaDB()
+		lm1.load('bnc-lemma.txt')
+		lm2.load('lemma.en.txt')
+		count1 = 0
+		count2 = 0
+		for stem in lm2.dump('stem'):
+			childs = lm2.get(stem)
+			stem = stem.lower()
+			if len(stem) <= 2 and stem.isupper():
+				continue
+			if not stem in lm1:
+				count1 += 1
+			else:
+				obj = lm1.get(stem)
+				for word in childs:
+					word = word.lower()
+					if not word in obj:
+						print '%s -> %s'%(stem, word)
+						count2 += 1
+			for word in childs:
+				lm1.add(stem, word.lower())
+		print 'count', count1, count2
+		lm1.save('lemma-bnc.txt')
+		return 0
+	test4()
+
+
+
+