mirror: 初始快照

bin/utils/check-valid-utf8.py

@@ -0,0 +1,13 @@
+import codecs
+import sys
+
+if __name__ == "__main__":
+    if len(sys.argv) < 1:
+        print("usage: check-valid-utf8 filemame")
+        exit(1)
+
+    f = open(sys.argv[1], encoding="utf-8", errors="strict")
+    for i, line in enumerate(f):
+        print(f"{i}: {line}")
+
+    print("Valid utf-8")

bin/utils/convert2dict.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Ngữ vựng Danh từ Thiền học" ./dict/Ngu-vung-Danh-tu-Thien-hoc.tab ./output/stardict/Ngu-vung-Danh-tu-Thien-hoc.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=zh --name="Phật Quang Đại từ điển Hán ngữ" ./dict/Phat-Quang-Dai-tu-dien-Han-ngu.tab ./output/stardict/Phat-Quang-Dai-tu-dien-Han-ngu.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Rộng mở tâm hồn" ./dict/Rong-mo-tam-hon.tab ./output/stardict/Rong-mo-tam-hon.ifo
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Anh Việt β" ./dict/TudienAnhVietBeta.tab ./output/stardict/TudienAnhVietBeta.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật Quang" ./dict/Tu-dien-Phat-Quang.tab ./output/stardict/Tu-dien-Phat-Quang.ifo
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Phật học Anh-Hán-Việt" ./dict/Tu-dien-Phat-hoc-Anh-Han-Viet.tab ./output/stardict/Tu-dien-Phat-hoc-Anh-Han-Viet.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật học Tinh tuyển" ./dict/Tu-dien-Phat-hoc-Tinh-tuyen.tab ./output/stardict/Tu-dien-Phat-hoc-Tinh-tuyen.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Minh Thông" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.tab ./output/stardict/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Thiện Phúc" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.tab ./output/stardict/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Đồng Loại" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.tab ./output/stardict/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Thiền Chửu" ./dict/TudienThienChuu.tab ./output/stardict/TudienThienChuu.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Tổng hợp Phật học" ./dict/Tu-dien-Tong-hop-Phat-hoc.tab ./output/stardict/Tu-dien-Tong-hop-Phat-hoc.ifo
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Đạo Uyển" ./dict/Tu-dien-Dao-Uyen.tab ./output/stardict/Tu-dien-Dao-Uyen.ifo
+
+rm -f ./output/all-stardict.zip
+zip -9 -j ./output/all-stardict.zip ./output/stardict/*.*

bin/utils/convert2epub.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Ngữ vựng Danh từ Thiền học" ./dict/Ngu-vung-Danh-tu-Thien-hoc.tab ./output/epub/Ngu-vung-Danh-tu-Thien-hoc.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=zh --name="Phật Quang Đại từ điển Hán ngữ" ./dict/Phat-Quang-Dai-tu-dien-Han-ngu.tab ./output/epub/Phat-Quang-Dai-tu-dien-Han-ngu.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Rộng mở tâm hồn" ./dict/Rong-mo-tam-hon.tab ./output/epub/Rong-mo-tam-hon.epub
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Anh Việt β" ./dict/TudienAnhVietBeta.tab ./output/epub/TudienAnhVietBeta.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật Quang" ./dict/Tu-dien-Phat-Quang.tab ./output/epub/Tu-dien-Phat-Quang.epub
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Phật học Anh-Hán-Việt" ./dict/Tu-dien-Phat-hoc-Anh-Han-Viet.tab ./output/epub/Tu-dien-Phat-hoc-Anh-Han-Viet.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật học Tinh tuyển" ./dict/Tu-dien-Phat-hoc-Tinh-tuyen.tab ./output/epub/Tu-dien-Phat-hoc-Tinh-tuyen.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Minh Thông" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.tab ./output/epub/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Thiện Phúc" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.tab ./output/epub/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Đồng Loại" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.tab ./output/epub/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Thiền Chửu" ./dict/TudienThienChuu.tab ./output/epub/TudienThienChuu.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Tổng hợp Phật học" ./dict/Tu-dien-Tong-hop-Phat-hoc.tab ./output/epub/Tu-dien-Tong-hop-Phat-hoc.epub
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Đạo Uyển" ./dict/Tu-dien-Dao-Uyen.tab ./output/epub/Tu-dien-Dao-Uyen.epub
+
+rm -f ./output/all-epub.zip
+zip -9 -j ./output/all-epub.zip ./output/epub/*.*

bin/utils/convert2kobo.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Ngữ vựng Danh từ Thiền học" ./dict/Ngu-vung-Danh-tu-Thien-hoc.tab ./output/kobo/Ngu-vung-Danh-tu-Thien-hoc.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=zh --name="Phật Quang Đại từ điển Hán ngữ" ./dict/Phat-Quang-Dai-tu-dien-Han-ngu.tab ./output/kobo/Phat-Quang-Dai-tu-dien-Han-ngu.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Rộng mở tâm hồn" ./dict/Rong-mo-tam-hon.tab ./output/kobo/Rong-mo-tam-hon.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Anh Việt β" ./dict/TudienAnhVietBeta.tab ./output/kobo/TudienAnhVietBeta.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật Quang" ./dict/Tu-dien-Phat-Quang.tab ./output/kobo/Tu-dien-Phat-Quang.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=en --target-lang=vi --name="Từ điển Phật học Anh-Hán-Việt" ./dict/Tu-dien-Phat-hoc-Anh-Han-Viet.tab ./output/kobo/Tu-dien-Phat-hoc-Anh-Han-Viet.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Phật học Tinh tuyển" ./dict/Tu-dien-Phat-hoc-Tinh-tuyen.tab ./output/kobo/Tu-dien-Phat-hoc-Tinh-tuyen.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Minh Thông" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.tab ./output/kobo/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Thiện Phúc" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.tab ./output/kobo/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=en --name="Từ điển Phật học Việt Anh - Đồng Loại" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.tab ./output/kobo/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Thiền Chửu" ./dict/TudienThienChuu.tab ./output/kobo/TudienThienChuu.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Tổng hợp Phật học" ./dict/Tu-dien-Tong-hop-Phat-hoc.tab ./output/kobo/Tu-dien-Tong-hop-Phat-hoc.kobo.zip &&
+pyglossary --read-format=Tabfile --source-lang=vi --target-lang=vi --name="Từ điển Đạo Uyển" ./dict/Tu-dien-Dao-Uyen.tab ./output/kobo/Tu-dien-Dao-Uyen.kobo.zip
+
+rm -f ./output/all-kobo.zip &&
+zip -9 -j ./output/all-kobo.zip ./output/kobo/*.*

bin/utils/convert2lingvo.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Ngữ vựng Danh từ Thiền học" ./dict/Ngu-vung-Danh-tu-Thien-hoc.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Chinese --dict-name "Phật Quang Đại từ điển Hán ngữ" ./dict/Phat-Quang-Dai-tu-dien-Han-ngu.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Rộng mở tâm hồn" ./dict/Rong-mo-tam-hon.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang English --to-lang Vietnamese --dict-name "Từ điển Anh Việt β" ./dict/TudienAnhVietBeta.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Từ điển Phật Quang" ./dict/Tu-dien-Phat-Quang.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang English --to-lang Vietnamese --dict-name "Từ điển Phật học Anh-Hán-Việt" ./dict/Tu-dien-Phat-hoc-Anh-Han-Viet.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Từ điển Phật học Tinh tuyển" ./dict/Tu-dien-Phat-hoc-Tinh-tuyen.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang English --dict-name "Từ điển Phật học Việt Anh - Minh Thông" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang English --dict-name "Từ điển Phật học Việt Anh - Thiện Phúc" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang English --dict-name "Từ điển Phật học Việt Anh - Đồng Loại" ./dict/Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Từ điển Thiền Chửu" ./dict/TudienThienChuu.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Từ điển Tổng hợp Phật học" ./dict/Tu-dien-Tong-hop-Phat-hoc.tab
+ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang Vietnamese --to-lang Vietnamese --dict-name "Từ điển Đạo Uyển" ./dict/Tu-dien-Dao-Uyen.tab
+
+mv *.dsl.dz ./output/lingvo/
+
+rm -f ./output/all-lingvo.zip
+zip -9 -j ./output/all-lingvo.zip ./output/lingvo/*.*

bin/utils/countwords.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+outfile="wordcount.txt"
+echo "Word count" > $outfile
+
+for file in *.tab
+do
+  wc -l $file >> $outfile
+done

bin/utils/create_all_stats.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+for filepath in $(ls ./ext-dict/*.tab)
+do
+  echo $filepath
+  python ./bin/tab_stats.py -i $filepath
+done

bin/utils/create_english_inflections.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+# from nltk.corpus import words
+from pattern.en import (
+    lexeme,  # conjugate, lemma,
+    pluralize,
+)
+
+# List of common English words
+wordlist = set(open("../misc/354984si.ngl").read().split())
+
+keys = set(open("english_keys.txt", encoding="utf-8").read().split())
+
+
+def getInflections(key):
+    inflections = set()
+    # print('"%s"' % key)
+
+    if key.isalpha():
+        try:
+            try:
+                lexeme(key)
+            except:
+                pass
+
+            inflections.add(lexeme(key))  # get all lexem inflections of words
+            inflections.add(pluralize(key))  # add plural inflections
+
+            inflections.intersection_update(wordlist)
+
+            print(inflections)
+
+        except:
+            pass
+            # print("Unexpected error")
+
+    return inflections
+
+
+keyfile = open("english_inflections.tab", "w", encoding="utf-8")
+
+try:
+    print(lexeme("be"))
+except:
+    pass
+
+print(lexeme("be"))
+print(lexeme("conclusion"))
+print(lexeme("harlot"))
+
+
+print(pluralize("be"))
+print(pluralize("conclusion"))
+print(pluralize("harlot"))
+
+for k in keys:
+    # print(lexeme(k))
+    # print(pluralize(k))
+
+    inflections = set()
+    # print('"%s"' % key)
+
+    inflections.update(lexeme(k))  # get all lexem inflections of words
+    inflections.add(pluralize(k))  # add plural inflections
+    if k in inflections:
+        inflections.remove(k)
+
+    inflections.intersection_update(wordlist)
+
+    # print(inflections)
+
+    if len(inflections):
+        keyfile.write("%s\t%s\n" % (k, "|".join(inflections)))
+
+keyfile.close()

bin/utils/createallhtml-inflections.sh

@@ -0,0 +1,2 @@
+./bin/createhtml.sh TudienAnhVietBeta-Inflections.txt TudienAnhVietBeta.tab
+./bin/createhtml.sh TudienThienChuu-Inflections.txt TudienThienChuu.tab

bin/utils/createallhtml.sh

@@ -0,0 +1,11 @@
+./bin/createhtml.sh NoInflections.txt Ngu-vung-Danh-tu-Thien-hoc.tab
+./bin/createhtml.sh NoInflections.txt Phat-Quang-Dai-tu-dien-Han-ngu.tab
+./bin/createhtml.sh NoInflections.txt Rong-mo-tam-hon.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Dao-Uyen.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-hoc-Anh-Han-Viet.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-hoc-Tinh-tuyen.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Phat-Quang.tab
+./bin/createhtml.sh NoInflections.txt Tu-dien-Tong-hop-Phat-hoc.tab

bin/utils/createallmobi.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Generate .mobi ebooks
+input_dir=$1
+output_dir=$2
+kindle_dir=kindle
+output_kindle_dir=$output_dir/$kindle_dir
+
+for fullpath in "*.opf"
+do
+  echo "Input $fullpath"
+  filename="${fullpath##*/}"                      # Strip longest match of */ from start
+  dir="${fullpath:0:${#fullpath} - ${#filename}}" # Substring from 0 thru pos of filename
+  base="${filename%.[^.]*}"                       # Strip shortest match of . plus at least one non-dot char from end
+  ext="${filename:${#base} + 1}" 
+  
+  echo "dir $dir"
+  echo "base $base"
+  echo "filename $filename"
+
+  wine ./bin/mobigen/mobigen.exe -unicode -s0 $fullpath
+
+  outfilename="$dir$base.mobi"
+  echo "outfilename: + $outfilename"
+  mv $outfilename $output_kindle_dir
+done
+
+#zip -9 -j $output_dir/all-kindle.zip $output_kindle_dir/*.mobi
+
+

bin/utils/createhtml.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Example: ./bin/createhtml.sh NoInflections.txt Ngu-vung-Danh-tu-Thien-hoc.tab
+# Run in root directory
+
+# echo $1
+# echo $2
+
+BASE="${2%.*}"
+
+# echo $BASE
+# echo "./dict/$BASE.opf.patch"
+
+# cd "/workspaces/tudien/"
+
+# Generates html and opf files
+python ./bin/tab2opf.py -s vi -t vi ./dict/$2 -i ./dict/$1
+
+# Move files to final directory
+mkdir -p ./dict/$BASE
+mv -f *.html ./dict/$BASE/
+mv -f *.opf ./dict/$BASE/
+
+# Apply patch to fix resultant .opf file
+#git apply "./dict/$BASE.opf.patch"
+

bin/utils/createmobi.sh

@@ -0,0 +1,5 @@
+# Example: ./bin/createmobi.sh Ngu-vung-Danh-tu-Thien-hoc
+
+wine ./bin/mobigen/mobigen.exe -unicode -s0 ./dict/$1/$1.opf
+
+mv ./dict/$1/$1.mobi ./output/kindle/

bin/utils/createpatch.sh

@@ -0,0 +1,5 @@
+# Use git log to get commit IDs
+git diff ab872f5..ea05b27 > mypatch.patch
+
+# Rename patch file accordingly
+git apply mypatch.patch

bin/utils/extract-definition.py

@@ -0,0 +1,218 @@
+"""Extract definition from existing Kindle dictionaries (For example https://akishop.com.vn/tong-hop-tu-dien-cho-kindle-nd49491.html)
+
+Steps
+1. If necessary, use DeDRM Calibre plugin (https://github.com/apprenticeharper/DeDRM_tools/) to remove DRM
+2. Use KindleUnpack -r to extract raw data from .prc/.mobi file
+3. Run following code to extract and cleanup definitions to a tab-seperated file format
+4. Now the output can be run through PyGlossary to convert to many dictionary formats
+"""
+
+import re
+from email.quoprimime import header_decode
+from html import unescape
+
+from bs4 import BeautifulSoup
+
+
+def striphtml(data):
+    p = re.compile(r"<.*?>")
+    return p.sub("", data)
+
+
+folder = r"./ext-dict/"
+
+FILE_DATA = {
+    "SPDict-Irregular-Verbs.html": {
+        "encoding": "utf-8",
+        "pattern": r"<i>(.+?)</i><br> @ <br>-- > (.+?) <br><mbp:pagebreak/><idx:entry>",
+        "remove": r"<[(/i|b|u)\s]*>",
+        "replace": [],
+        "skips": 0,
+    },
+    "Wikipedia/mobi7/Wikipedia.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h1>(.+?)<\/h1>(.+?)<br>",
+        "remove": r"<[(/i|b|u)\s]*>",
+        "replace": [],
+        "skips": 0,
+    },
+    "Bach khoa toan thu/mobi7/Bach khoa toan thu.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h1>(.+?)<\/h1>(.+?)<mbp:pagebreak\/>",
+        "remove": r"<[(/i|b|u|p|span|font|em|div)\s].*?>",
+        "replace": [],
+        "skips": 0,
+    },
+    "Duc - Viet - Ho Ngoc Duc/mobi7/Duc - Viet - Ho Ngoc Duc.rawml": {
+        "encoding": "cp1252",
+        "pattern": r"<h3>(.+?)</h3>(.+?)<br/> <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"<[(/i|b|u)\s]*>",
+        "skips": 0,
+    },
+    "Dictionary of Computing/mobi7/Dictionary of Computing.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h3>(.+?)</h3>(.+?)<mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"",
+        "skips": 4,
+    },
+    "Nhat - Viet/mobi7/Nhat - Viet.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h2>(.+?)</h2>(.+?)<mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"",
+        "skips": 2,
+    },
+    "Phap-Viet - Ho Ngoc Duc/mobi7/Phap-Viet - Ho Ngoc Duc.rawml": {
+        "encoding": "cp1252",
+        "pattern": r"<h3>(.+?)</h3>(.+?)<br/> <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"<[(/i|b|u)\s]*>",
+        "skips": 0,
+    },
+    "Tieng Viet - Ho Ngoc Duc/mobi7/Tieng Viet - Ho Ngoc Duc.rawml": {
+        "encoding": "cp1252",
+        "pattern": r"<h3>(.+?)</h3>(.+?) <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"<[(/i|b|u)\s]*>",
+        "skips": 0,
+    },
+    "WordNet 2.0/mobi7/WordNet 2.0.rawml": {
+        "encoding": "cp1252",
+        "pattern": r"<h3>(.+?)</h3>(.+?) <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"<[(/i|b|u)\s]*>",
+        "skips": 0,
+    },
+    "Viet - Nhat NXBVHTT/mobi7/Viet - Nhat NXBVHTT.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h2>(.+?)<\/h2>(.+?) <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r"",
+        "skips": 0,
+    },
+    "britannica_com - Unknown/mobi7/britannica_com - Unknown.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<h2>(.+?)<\/h2>(.+?) <mbp:pagebreak/>",
+        "replace": [],
+        "remove": r'<[(/i|b|u)\s]*>|<a href="http:\/\/www\.babylon\.com\/redirects\/redir\.cgi\?type=britannica&query=.+?">Learn more at Britannica\.com<\/a>',
+        "skips": 0,
+    },
+    "Tu dien Han ngu - hhh/mobi7/Tu dien Han ngu - hhh.rawml": {
+        "encoding": "utf-8",
+        "pattern": r'<font size="7">(.+?)<\/font>(.+?)<mbp:pagebreak\/>',
+        "replace": [],
+        "remove": r"",
+        "skips": 0,
+        "strip_html": True,
+    },
+    "SPDict-English-Grammar.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<i>(.+?)</i><br>(.+?)<br><mbp:pagebreak/><idx:entry>",
+        "remove": r"<[(/i|b|u)\s]*>",
+        "replace": [("<a filepos=XXXXXXXXXX >", "")],
+        "skips": 0,
+    },
+    "SPDict-Viet-Anh.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<i>.+?</i>(.+?)<br>(.+?)<mbp:pagebreak/><idx:entry>",
+        #        'pattern': r'<i>.+?</i><br> <b>(.+?)</b><br>(.+?)<br><mbp:pagebreak/><idx:entry>',
+        "remove": r"<[(/i|b|u|br)\s]*>",
+        "replace": [],
+        "skips": 0,
+    },
+    "SPDict-Anh-Viet.rawml": {
+        "encoding": "utf-8",
+        "pattern": r"<i>(.+?)</i>(.+?)<mbp:pagebreak/>",
+        "remove": r"<[(/i|b|u|br)\s]*>",
+        "replace": [('<font color="#FF0000"> ', " ")],
+        "skips": 0,
+    },
+}
+
+file = r"SPDict-Anh-Viet.rawml"
+
+filepath = folder + file
+
+print(f"Parsing {file}")
+
+file_encoding = FILE_DATA[file]["encoding"]
+skips = FILE_DATA[file]["skips"]
+
+DEBUG = False
+
+word_mark = "<idx:orth"
+
+with open(filepath, encoding=file_encoding) as f:
+    html = f.read()
+
+    # HTML entities need to unescape to create Unicode chars
+    if file_encoding == "cp1252":
+        html = unescape(html)
+
+    # Print the extracted data
+
+    # outhtml = remove_tags(html, file)
+    # print(outhtml)
+
+    pattern = re.compile(FILE_DATA[file]["pattern"])
+
+    with open(filepath.replace(".rawml", ".tab"), "w", encoding="utf-8") as out:
+        count = 1
+        print("Start matching...")
+
+        print(f"Head word marks: {html.count(word_mark)}")
+
+        for match in pattern.finditer(html):
+            if count <= skips:
+                continue
+
+            headword = match.group(1).strip()
+            definition = match.group(2).strip()
+
+            definition = definition.replace(f"@{headword}", "")
+
+            # StarDict metadata
+            if headword.find("00-database") >= 0:
+                continue
+
+            definition = definition.replace("\t", " ")
+
+            if DEBUG:
+                print(f"{headword}\t{definition}\n\n")
+
+            definition = re.sub(FILE_DATA[file]["remove"], "", definition)
+            headword = re.sub(FILE_DATA[file]["remove"], "", headword)
+
+            for org, new in FILE_DATA[file]["replace"]:
+                headword = headword.replace(org, new)
+
+            for org, new in FILE_DATA[file]["replace"]:
+                definition = definition.replace(org, new)
+
+            headword = headword.strip()
+            definition = definition.strip()
+
+            if "recover_mark" in FILE_DATA[file] and headword.find("[VV]") >= 0:
+                pos = definition.find("-")
+                headword = definition[:pos]
+                definition = definition[pos + 2 :]
+
+            if "strip_html" in FILE_DATA[file] and FILE_DATA[file]["strip_html"]:
+                definition = striphtml(definition)
+
+            if DEBUG:
+                print(f"{headword}\t{definition}\n\n")
+
+            if not headword or not definition:
+                print(f"Warning: something is empty |{headword}|\t|{definition}|")
+                continue
+
+            count += 1
+
+            out.writelines(f"{headword}\t{definition}\n")
+
+    print(f"No. of items: {count}")
+
+print(f"Done")

bin/utils/stardict2tab.sh

@@ -0,0 +1,18 @@
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/es-vi/star_tbnviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/fr-vi/star_phapviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-en/star_vietanh
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/ru-vi/star_ngaviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-fr/star_vietphap
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/no-vi/star_nauyviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-de/star_vietduc
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/zh-vi/star_trungviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/pt-vi/star_bdnviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/ja-vi/star_nhatviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/de-vi/star_ducviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-ru/star_vietnga
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/cz-vi/star_secviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/it-vi/star_yviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/en-vi/star_anhviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-es/star_viettbn
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/ko-vi/star_hanviet
+./bin/stardict2txt.py --cleanup  ./ext-stardict-vi/vi-ja/star_vietnhat

bin/utils/stardict2txt.py

@@ -0,0 +1,422 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import gzip
+import struct
+import sys
+
+CLEANUP = False
+
+
+class IfoFileException(Exception):
+    """Exception while parsing the .ifo file.
+    Now version error in .ifo file is the only case raising this exception.
+    """
+
+    def __init__(self, description="IfoFileException raised"):
+        """Constructor from a description string.
+
+        Arguments:
+        - `description`: a string describing the exception condition.
+        """
+        self._description = description
+
+    def __str__(self):
+        """__str__ method, return the description of exception occured."""
+        return self._description
+
+
+class IfoFileReader(object):
+    """Read infomation from .ifo file and parse the infomation a dictionary.
+    The structure of the dictionary is shown below:
+    {key, value}
+    """
+
+    def __init__(self, filename):
+        """Constructor from filename.
+
+        Arguments:
+        - `filename`: the filename of .ifo file of stardict.
+        May raise IfoFileException during initialization.
+        """
+        self._ifo = dict()
+        with open(filename, "r") as ifo_file:
+            self._ifo["dict_title"] = ifo_file.readline().strip()  # dictionary title
+            line = ifo_file.readline()  # version info
+            key, equal, value = line.partition("=")
+            key = key.strip()
+            value = value.strip()
+            # check version info, raise an IfoFileException if error encounted
+            if key != "version":
+                raise IfoFileException("Version info expected in the second line of {!r:s}!".format(filename))
+            if value != "2.4.2" and value != "3.0.0":
+                raise IfoFileException("Version expected to be either 2.4.2 or 3.0.0, but {!r:s} read!".format(value))
+            self._ifo[key] = value
+            # read in other infomation in the file
+            # all values are all string
+            for line in ifo_file:
+                key, equal, value = line.partition("=")
+                key = key.strip()
+                value = value.strip()
+                self._ifo[key] = value
+            # check if idxoffsetbits should be discarded due to version info
+            if self._ifo["version"] == "3.0.0" and "idxoffsetbits" in self._ifo:
+                del self._ifo["version"]
+
+    def get_ifo(self, key):
+        """Get configuration value.
+
+        Arguments:
+        - `key`: configuration option name
+        Return:
+        - configuration value corresponding to the specified key if exists, otherwise False.
+        """
+        if key not in self._ifo:
+            return False
+        return self._ifo[key]
+
+    def dump(self):
+        """Debug function"""
+        for k, v in self._ifo.iteritems():
+            if type(v) is not str:
+                v = str(v)
+            print("%s: %s" % (k, str(v)))
+
+
+class IdxFileReader(object):
+    """Read dictionary indexes from the .idx file and store the indexes in a list and a dictionary.
+    The list contains each entry in the .idx file, with subscript indicating the entry's origin index in .idx file.
+    The dictionary is indexed by word name, and the value is an integer or a list of integers pointing to
+    the entry in the list.
+    """
+
+    def __init__(self, filename, compressed=False, index_offset_bits=32):
+        """
+
+        Arguments:
+        - `filename`: the filename of .idx file of stardict.
+        - `compressed`: indicate whether the .idx file is compressed.
+        - `index_offset_bits`: the offset field length in bits.
+        """
+        if compressed:
+            with gzip.open(filename, "rb") as index_file:
+                self._content = index_file.read()
+        else:
+            with open(filename, "rb") as index_file:
+                self._content = index_file.read()
+        self._offset = 0
+        self._index = 0
+        self._index_offset_bits = index_offset_bits
+        self._word_idx = dict()
+        self._index_idx = list()
+        for word_str, word_data_offset, word_data_size, index in self:
+            self._index_idx.append((word_str, word_data_offset, word_data_size))
+            if word_str in self._word_idx:
+                if isinstance(self._word_idx[word_str], list):
+                    self._word_idx[word_str].append(len(self._index_idx) - 1)
+                else:
+                    self._word_idx[word_str] = [self._word_idx[word_str], len(self._index_idx) - 1]
+            else:
+                self._word_idx[word_str] = len(self._index_idx) - 1
+        del self._content
+        del self._index_offset_bits
+        del self._index
+
+    def __iter__(self):
+        """Define the iterator interface."""
+        return self
+
+    def __next__(self):
+        # py3 call to py2 interface - so as to support both
+        return self.next()
+
+    def next(self):
+        """Define the iterator interface."""
+        if self._offset == len(self._content):
+            raise StopIteration
+        word_data_offset = 0
+        word_data_size = 0
+        end = self._content.find(b"\0", self._offset)
+        # word_str process
+        word_str = self._content[self._offset : end]
+        self._offset = end + 1
+        # word_data_offset
+        if self._index_offset_bits == 64:
+            (word_data_offset,) = struct.unpack("!I", self._content[self._offset : self._offset + 8])
+            self._offset += 8
+        elif self._index_offset_bits == 32:
+            (word_data_offset,) = struct.unpack("!I", self._content[self._offset : self._offset + 4])
+            self._offset += 4
+        else:
+            raise ValueError
+        # word_data_size
+        (word_data_size,) = struct.unpack("!I", self._content[self._offset : self._offset + 4])
+        self._offset += 4
+        self._index += 1
+        return (word_str, word_data_offset, word_data_size, self._index)
+
+    def get_index_by_num(self, number):
+        """Get index infomation of a specified entry in .idx file by origin index.
+        May raise IndexError if number is out of range.
+
+        Arguments:
+        - `number`: the origin index of the entry in .idx file
+        Return:
+        A tuple in form of (word_str, word_data_offset, word_data_size)
+        """
+        if number >= len(self._index_idx):
+            raise IndexError("Index out of range! Acessing the {:d} index but totally {:d}".format(number, len(self._index_idx)))
+        return self._index_idx[number]
+
+    def get_index_by_word(self, word_str):
+        """Get index infomation of a specified word entry.
+
+        Arguments:
+        - `word_str`: name of word entry.
+        Return:
+        Index infomation corresponding to the specified word if exists, otherwise False.
+        The index infomation returned is a list of tuples, in form of [(word_data_offset, word_data_size) ...]
+        """
+        if word_str not in self._word_idx:
+            return False
+        number = self._word_idx[word_str]
+        index = list()
+        if isinstance(number, list):
+            for n in number:
+                index.append(self._index_idx[n][1:])
+        else:
+            index.append(self._index_idx[number][1:])
+        return index
+
+    def dump_word(self):
+        for word_str in index._word_idx:
+            print(word_str, ": ", index.get_index_by_word(word_str))
+
+
+class SynFileReader(object):
+    """Read infomation from .syn file and form a dictionary as below:
+    {synonym_word: original_word_index}, in which 'original_word_index' could be a integer or
+    a list of integers.
+
+    """
+
+    def __init__(self, filename):
+        """Constructor.
+
+        Arguments:
+        - `filename`: The filename of .syn file of stardict.
+        """
+        self._syn = dict()
+        with open(filename, "rb") as syn_file:
+            content = syn_file.read()
+        offset = 0
+        while offset < len(content):
+            end = content.find("\0", offset)
+            synonym_word = content[offset:end]
+            offset = end
+            original_word_index = struct.unpack("!I", content[offset, offset + 4])
+            offset += 4
+            if synonym_word in self._syn:
+                if isinstance(self._syn[synonym_word], list):
+                    self._syn[synonym_word].append(original_word_index)
+                else:
+                    self._syn[synonym_word] = [self._syn[synonym_word], original_word_index]
+            else:
+                self._syn[synonym_word] = original_word_index
+
+    def get_syn(self, synonym_word):
+        """
+
+        Arguments:
+        - `synonym_word`: synonym word.
+        Return:
+        If synonym_word exists in the .syn file, return the corresponding indexes, otherwise False.
+        """
+        if synonym_word not in self._syn:
+            return False
+        return self._syn[synonym_word]
+
+
+class DictFileReader(object):
+    """Read the .dict file, store the data in memory for querying."""
+
+    def __init__(self, filename, dict_ifo, dict_index, compressed=False):
+        """Constructor.
+
+        Arguments:
+        - `filename`: filename of .dict file.
+        - `dict_ifo`: IfoFileReader object.
+        - `dict_index`: IdxFileReader object.
+        """
+        self._dict_ifo = dict_ifo
+        self._dict_index = dict_index
+        self._compressed = compressed
+        self._offset = 0
+        if self._compressed:
+            with gzip.open(filename, "rb") as dict_file:
+                self._dict_file = dict_file.read()
+        else:
+            with open(filename, "rb") as dict_file:
+                self._dict_file = dict_file.read()
+
+    def get_dict_by_word(self, word):
+        """Get the word's dictionary data by it's name.
+
+        Arguments:
+        - `word`: word name.
+        Return:
+        The specified word's dictionary data, in form of dict as below:
+        {type_identifier: infomation, ...}
+        in which type_identifier can be any character in "mlgtxykwhnrWP".
+        """
+        result = list()
+        indexes = self._dict_index.get_index_by_word(word)
+        if indexes == False:
+            return False
+        sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
+        for index in indexes:
+            self._offset = index[0]
+            size = index[1]
+            if sametypesequence:
+                result.append(self._get_entry_sametypesequence(size))
+            else:
+                result.append(self._get_entry(size))
+        return result
+
+    def dump(self, save_file):
+        """ "dump all word"""
+        with open(save_file, "wb+") as f:
+            for w in self._dict_index._word_idx:
+                meaning_lst = self.get_dict_by_word(w)
+                # print('--------------------------------')
+                # print(w)
+                # print(self.get_dict_by_word(w))
+                # print('--------------------------------')
+                # print(repr(w))
+                # out_encoding = 'utf-8'  # force/assume encoding to UTF8, no config
+
+                if w.find(b"00-database") >= 0:  # Skips line with StarDict metadata
+                    continue
+
+                f.write(w)
+                f.write(b"\t")
+                for i, m in enumerate(meaning_lst):
+                    meaning_bytes = b" ".join(m.values())
+
+                    if CLEANUP:  # Skip first meaning if CLEAN is needed
+                        if meaning_bytes.find(b"@") == 0:  # Start with @ means a repetition
+                            meaning_bytes = meaning_bytes[meaning_bytes.find(b"\n") + 1 :]
+
+                    meaning_bytes = meaning_bytes.replace(b"\r", b" ")  # replace
+                    meaning_bytes = meaning_bytes.replace(b"\n", b" ")  # replace
+                    # meaning_bytes = meaning_bytes.replace(b'\n', b'')  # remove
+
+                    f.write(meaning_bytes)
+
+                f.write(b"\n")
+
+        print(f"Number of words written: {len(self._dict_index._word_idx)}")
+
+    def get_dict_by_index(self, index):
+        """Get the word's dictionary data by it's index infomation.
+
+        Arguments:
+        - `index`: index of a word entrt in .idx file.'
+        Return:
+        The specified word's dictionary data, in form of dict as below:
+        {type_identifier: infomation, ...}
+        in which type_identifier can be any character in "mlgtxykwhnrWP".
+        """
+        word, offset, size = self._dict_index.get_index_by_num(index)
+        self._offset = offset
+        sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
+        if sametypesequence:
+            return self._get_entry_sametypesequence(size)
+        else:
+            return self._get_entry(size)
+
+    def _get_entry(self, size):
+        result = dict()
+        read_size = 0
+        start_offset = self._offset
+        while read_size < size:
+            (type_identifier,) = struct.unpack("!c", self._dict_file[self._offset : self._offset + 1])
+            self._offset += 1
+            # type_identifier = str(type_identifier)
+            # print(type_identifier)
+            if type_identifier in "mlgtxykwhnr":
+                result[type_identifier] = self._get_entry_field_null_trail()
+            else:
+                result[type_identifier] = self._get_entry_field_size()
+            read_size = self._offset - start_offset
+        return result
+
+    def _get_entry_sametypesequence(self, size):
+        start_offset = self._offset
+        result = dict()
+        sametypesequence = self._dict_ifo.get_ifo("sametypesequence")
+        for k in range(0, len(sametypesequence)):
+            # the last field has no tailing '\0'
+            if sametypesequence[k] in "mlgtxykwhnr":
+                if k == len(sametypesequence) - 1:
+                    result[sametypesequence[k]] = self._get_entry_field_size(size - (self._offset - start_offset))
+                else:
+                    result[sametypesequence[k]] = self._get_entry_field_null_trail()
+            elif sametypesequence[k] in "WP":
+                if k == len(sametypesequence) - 1:
+                    result[sametypesequence[k]] = self._get_entry_field_size(size - (self._offset - start_offset))
+                else:
+                    result[sametypesequence[k]] = self._get_entry_field_size()
+        return result
+
+    def _get_entry_field_null_trail(self):
+        end = self._dict_file.find("\0", self._offset)
+        result = self._dict_file[self._offset : end]
+        self._offset = end + 1
+        return result
+
+    def _get_entry_field_size(self, size=None):
+        # for the 'W' 'P' case
+        if size is None:
+            (size,) = struct.unpack("!I", self._dict_file[self._offset : self._offset + 4])
+            self._offset += 4
+        result = self._dict_file[self._offset : self._offset + size]
+        self._offset += size
+        return result
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 1:
+        print("usage: stardict2txt pathbase")
+        print("Note that the pathbase doesn't include file extension.")
+        exit(1)
+
+    parser = argparse.ArgumentParser(description="Convert StarDict dictionary to tab seperated file", usage="Usage: python stardict2txt --input basename")
+
+    parser.add_argument("-c", "--cleanup", action=argparse.BooleanOptionalAction)
+    parser.add_argument("basename", help="Basename of StarDict dictionary files")
+
+    args, array = parser.parse_known_args()
+    cleanup = args.cleanup
+    CLEANUP = cleanup
+
+    dict_name = args.basename
+    ifo_file = dict_name + ".ifo"
+    idx_file = dict_name + ".idx"
+    dict_file = dict_name + ".dict.dz"
+    text_file = dict_name + ".tab"
+
+    print(f"Start conversion: {ifo_file}")
+    # info read test done
+    info = IfoFileReader(ifo_file)
+    # info.dump()
+
+    # index read test
+    index = IdxFileReader(idx_file)
+    # index.dump_word()
+
+    # dict test
+    dict_reader = DictFileReader(dict_file, info, index, True)
+    dict_reader.dump(text_file)
+
+    print("Done.")

bin/utils/tab_stats.py

@@ -0,0 +1,162 @@
+"""Statistics for the dictionary data file"""
+
+import argparse
+import collections
+import pathlib
+import string
+
+
+def frequency(l):
+    cp = []
+    maxl = []
+    minl = []
+
+    ctr = collections.Counter(l)
+
+    print(ctr.most_common())
+    for i in ctr:
+        r = ctr[i]
+        cp.append(r)
+    p = max(cp)
+    q = min(cp)
+    # print(cp)
+    for i in ctr:
+        if ctr[i] == p:
+            maxl.append(i)
+        if ctr[i] == q:
+            minl.append(i)
+
+    return (sorted(minl), sorted(maxl))
+
+
+COUNT = 10
+DEF_LEN_DIV = 100
+MAX_LINE = 1000  # Run multitple round to avoid loading a huge file
+
+
+def list_words(text):
+    return [word.strip(string.punctuation + "\n") for word in text.split()]
+
+
+def writeStats(file, contents):
+    print(str(contents))
+    file.write(f"{str(contents)}\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Convert all dictionaries in a folder", usage="Usage: python tab_stats.py --input data.tab --output data.stats"
+    )
+
+    parser.add_argument("-i", "--input", required=True, help="Input folder containing .tsv and .dfo files")
+    parser.add_argument("-o", "--output", help="Output folder containing dictionary files")
+
+    args, array = parser.parse_known_args()
+    inputfile = args.input
+    ouputfile = args.output
+
+    mainCharCounter = collections.Counter()
+    mainWordCounter = collections.Counter()
+    mainHeadwordCounter = collections.Counter()
+    mainDefinitionCounter = collections.Counter()
+
+    # Stats về chiều dài headword, definition
+    # Tách lượt để thống kê không thì hết bộ nhớ
+
+    inputpath = pathlib.Path(inputfile)
+
+    if not ouputfile:
+        ouputpath = inputpath.with_suffix(".stats")
+    else:
+        ouputpath = pathlib.Path(ouputfile)
+
+    try:
+        infile = open(inputpath, encoding="utf-8", errors="strict")
+
+        outfile = open(ouputpath, "w", encoding="utf-8")
+
+        line = infile.readline()
+
+        line_count = 1
+        issue_count = 0
+        lines = []
+        text = ""
+        words = []
+
+        writeStats(outfile, "# File Checks")
+
+        while line:
+            # print(f'Line {line_count}')
+
+            #            lines.append(line)
+
+            mainCharCounter = mainCharCounter + collections.Counter(line)
+            # mainWordCounter = mainWordCounter + collections.Counter(list_words(line))
+
+            items = line.strip().split("\t")
+            no_items = len(items)
+
+            if no_items != 2:
+                writeStats(outfile, f"### Error on line {line_count}: Wrong number of items ({no_items})> {line.strip()}")
+                issue_count += 1
+
+            else:
+                mainHeadwordCounter.update([len(items[0])])
+                mainDefinitionCounter.update([len(items[1]) // DEF_LEN_DIV * DEF_LEN_DIV])
+                pass
+
+            line_count += 1
+            line = infile.readline()
+
+            if not line or ((line_count % MAX_LINE) == 0):  # Every MAX_LINE lines, accomulate stats
+                print(f"Line {line_count}")
+                # mainCharCounter = mainCharCounter + collections.Counter(text)
+                # mainWordCounter = mainWordCounter + collections.Counter(words)
+                # text = ''
+                # lines = []
+                # words = []
+                outfile.flush()
+
+        writeStats(outfile, "# Line stats")
+        writeStats(outfile, f"## Line count: {line_count}")
+        writeStats(outfile, f"## Issue count: {issue_count}")
+
+        writeStats(outfile, "# Character frequency")
+        writeStats(outfile, str(mainCharCounter.most_common(COUNT)))
+        writeStats(outfile, str(mainCharCounter.most_common()[: -COUNT - 1 : -1]))
+
+        # writeStats(outfile, '# Word frequency')
+        # writeStats(outfile, str(mainWordCounter.most_common(COUNT)))
+        # writeStats(outfile, str(mainWordCounter.most_common()[:-COUNT-1:-1]))
+
+        writeStats(outfile, "# Headword len frequency")
+        writeStats(outfile, str(mainHeadwordCounter.most_common(COUNT)))
+        writeStats(outfile, str(mainHeadwordCounter.most_common()[: -COUNT - 1 : -1]))
+
+        writeStats(outfile, "# Definition len frequency")
+        writeStats(outfile, str(mainDefinitionCounter.most_common(COUNT)))
+        writeStats(outfile, str(mainDefinitionCounter.most_common()[: -COUNT - 1 : -1]))
+
+    except IOError as err:
+        print(f"Error reading file: {err}")
+
+    except UnicodeDecodeError as err:
+        print(f"Error decoding file: {err}")
+
+        # try:
+        # print('Write file to compare')
+        # infile = open(inputpath, encoding='utf-8', errors='replace')
+        # lines = infile.readlines()
+        # infile.close()
+
+        # outfile = open(str(inputpath) + '.txt', 'w', encoding='utf-8')
+        # outfile.writelines(lines)
+        # outfile.close()
+
+        # except IOError as err1:
+        # print(f'Error reading file: {err1}')
+        # exit(1)
+
+
+if __name__ == "__main__":
+    main()

bin/utils/taotudienphathoc.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import json
+import os
+
+filepath = "./dict/Tu-dien-Tong-hop-Phat-hoc.json"
+fileout = "./dict/Tu-dien-Tong-hop-Phat-hoc.tab.new"
+
+dictdata = json.load(open(filepath, "rb"))
+
+print(dictdata.keys())
+
+entries = dictdata["WordRaw"]
+print("Num of entries: {}".format(len(entries)))
+
+dict_names = {
+    1: {"Name": "Từ điển Phật Quang", "File": "Tu-dien-Phat-Quang"},
+    2: {"Name": "Từ điển Phật học Việt Anh - Thiện Phúc", "File": "Tu-dien-Phat-hoc-Viet-Anh-Thien-Phuc"},
+    3: {"Name": "Từ điển Phật học Anh-Hán-Việt", "File": "Tu-dien-Phat-hoc-Anh-Han-Viet"},
+    4: {"Name": "Ngữ vựng Danh từ Thiền học", "File": "Ngu-vung-Danh-tu-Thien-hoc"},
+    5: {"Name": "Từ điển Đạo Uyển", "File": "Tu-dien-Dao-Uyen"},
+    6: {"Name": "Từ điển Phật học Việt Anh - Đồng Loại", "File": "Tu-dien-Phat-hoc-Viet-Anh-Dong-Loai"},
+    7: {"Name": "Từ điển Phật học Việt Anh - Minh Thông", "File": "Tu-dien-Phat-hoc-Viet-Anh-Minh-Thong"},
+    8: {"Name": "Phật Quang Đại từ điển Hán ngữ", "File": "Phat-Quang-Dai-tu-dien-Han-ngu"},
+    9: {"Name": "Rộng mở tâm hồn", "File": "Rong-mo-tam-hon"},
+    10: {"Name": "Từ điển Phật học Tinh tuyển", "File": "Tu-dien-Phat-hoc-Tinh-tuyen"},
+}
+
+dictionaries = {}
+
+use_dicts = {
+    1,
+    2,
+    4,
+    5,
+    9,
+    10,
+}  # 3, 6, 7, 8}
+
+count = 0
+with open(fileout, "w", encoding="utf-8") as f:
+    for i in entries:
+        key = i["Word"].strip()
+        meaning = i["Mean"].strip()
+
+        if key.find("\n") >= 0:
+            key = key.replace("\r\n", "|")
+
+        if meaning.find("\n") >= 0:
+            meaning = meaning.replace("\r\n", "|")
+
+        dk = i["Dict"]
+
+        if dk not in dictionaries:
+            dictionaries[dk] = []
+
+        dictionaries[dk].append({"Word": key, "Mean": meaning})
+
+        if len(meaning) and dk in use_dicts:
+            f.write(f"{key}\t{meaning} [{dict_names[dk]['Name']}]\n")
+
+            count = count + 1
+
+print(dictionaries.keys())
+
+for key in dictionaries:
+    print("%i\t%i\t%s" % (len(dictionaries[key]), key, dict_names[key]["Name"]))
+
+    filename = f"./dict/{dict_names[key]['File']}.tab"
+
+    with open(filename, "w", encoding="utf-8") as file:
+        for e in dictionaries[key]:
+            file.write(f"{e['Word']}\t{e['Mean']}\n")
+
+print(f"{count} words processed")

bin/utils/useful.sh

@@ -0,0 +1,6 @@
+for f in "./ext-dict/*.tab" do ./bin/tab_stats.py $f done
+
+mv ext-stardict-vi/*/*.tab ./ext-dict
+
+cat /workspaces/tudien/dict/TudienAnhVietBeta.tab /workspaces/tudien/ext-dict/star_vietanh.tab > /workspaces/tudien/dict/TudienAnhVietAnh.tab
+