380 lines
13 KiB
Python
Executable File
380 lines
13 KiB
Python
Executable File
"""Generates summary of available dictionaries"""
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
|
|
import langcodes
|
|
|
|
# Prefer the stdlib tomllib (Py3.11+) but fall back to tomli if necessary
|
|
try:
|
|
import tomllib as toml
|
|
except Exception:
|
|
import tomli as toml
|
|
|
|
DOWNLOAD_TAG = "v4.4" # Set the GitHub tag version that asscociates with the release
|
|
|
|
|
|
def parse_toml_file(toml_path):
|
|
"""Parse the .toml file and return its metadata as a dictionary.
|
|
|
|
This maps the common keys found in the repository's .toml files to the
|
|
metadata shape expected by the rest of the script.
|
|
"""
|
|
metadata = {
|
|
"Name": "",
|
|
"Description": "",
|
|
"Source": "",
|
|
"Target": "",
|
|
"Owner_Editor": "",
|
|
"URL": "",
|
|
"Num_entries": 0,
|
|
"Version": "",
|
|
"Inflections": "",
|
|
}
|
|
|
|
try:
|
|
# tomllib/tomli expect a binary file object for load()
|
|
with open(toml_path, "rb") as fh:
|
|
doc = toml.load(fh)
|
|
|
|
for key in metadata.keys():
|
|
if key in doc:
|
|
metadata[key] = doc.get(key, metadata[key])
|
|
|
|
# # map known keys from the toml file to the metadata dict
|
|
# metadata["Name"] = doc.get("Name", "")
|
|
# metadata["Description"] = doc.get("Description", "")
|
|
# metadata["Source"] = doc.get("Source", "")
|
|
# metadata["Target"] = doc.get("Target", "")
|
|
# metadata["Num_entries"] = doc.get("Num_entries", "")
|
|
# metadata["Owner_Editor"] = doc.get("Owner_Editor", doc.get("Owner_Editor", ""))
|
|
# metadata["URL"] = doc.get("URL", "")
|
|
# metadata["Version"] = doc.get("Version", "")
|
|
except FileNotFoundError:
|
|
print(f"Error: {toml_path} not found.")
|
|
except Exception as exc:
|
|
print(f"Error parsing TOML file {toml_path}: {exc}")
|
|
|
|
return metadata
|
|
|
|
|
|
def count_lines_in_tab(tab_path):
|
|
"""Count the number of lines in the corresponding .tab file."""
|
|
try:
|
|
with open(tab_path, "r", encoding="utf-8") as file:
|
|
return sum(1 for _ in file)
|
|
except FileNotFoundError:
|
|
print(f"Error: {tab_path} not found.")
|
|
return 0
|
|
|
|
|
|
SUPPORTED_EXTENSIONS = {
|
|
# Extension: {Dir: "dir", "name": "Name"},
|
|
"dictd.zip": {"dir": "dictd", "name": "DictD"},
|
|
"dsl.dz": {"dir": "lingvo", "name": "Lingvo (DSL)"},
|
|
"epub": {"dir": "epub", "name": "EPUB"},
|
|
"kobo.zip": {"dir": "kobo", "name": "Kobo"},
|
|
"mobi": {"dir": "kindle", "name": "Kindle (.mobi)"},
|
|
"stardict.zip": {"dir": "stardict", "name": "StartDict"},
|
|
"yomitan.zip": {"dir": "yomitan", "name": "Yomitan"},
|
|
"mdx": {"dir": "mdict", "name": "MDict"},
|
|
"pleco.zip": {"dir": "pleco", "name": "Pleco"},
|
|
"slob": {"dir": "slob", "name": "Aard 2"},
|
|
"dic": {"dir": "xdxf", "name": "Pocketbook"},
|
|
# "xdxf.zip": {"dir": "xdxf", "name": "XDXF"},
|
|
}
|
|
|
|
COLUMNS = {
|
|
"Name": "Name",
|
|
"Description": "Description",
|
|
"Source": "Source",
|
|
"Target": "Target",
|
|
"Owner_Editor": "Owner_Editor",
|
|
"Url": "URL",
|
|
"Version": "Version",
|
|
"Num_entries": "Num_entries",
|
|
}
|
|
|
|
|
|
def get_downloadable_files(filebase, tag_download, output_dir):
|
|
"""Return the downloadable file links based on the filebase name."""
|
|
download_links = {}
|
|
|
|
for ext in SUPPORTED_EXTENSIONS:
|
|
# Generate the URL for the downloadable file
|
|
download_url = f"https://github.com/catusf/tudien/releases/download/{tag_download}/{filebase}.{ext}"
|
|
|
|
download_path = os.path.join(output_dir, f"{filebase}.{ext}")
|
|
|
|
# print(f"*** Checking for {download_path}")
|
|
|
|
if os.path.exists(os.path.join(output_dir, f"{filebase}.{ext}")):
|
|
download_links[ext] = download_url
|
|
else:
|
|
download_links[ext] = ""
|
|
|
|
return download_links
|
|
|
|
|
|
def generate_summary(dict_dir, output_dir, showfull):
|
|
"""Generate a list of dictionaries containing metadata for each .toml file."""
|
|
print(f"Generating summary data for {dict_dir}")
|
|
|
|
print(f"Input directory: {dict_dir}\nOutput directory: {output_dir}")
|
|
|
|
data = []
|
|
needed_files = []
|
|
num_dict_found = 0
|
|
|
|
pleco_data = set()
|
|
|
|
for filename in os.listdir(dict_dir):
|
|
if filename.endswith(".txt"):
|
|
filebase = filename[:-4]
|
|
pleco_data.add(filebase)
|
|
|
|
for filename in os.listdir(dict_dir):
|
|
if not filename.endswith(".toml"):
|
|
continue
|
|
filebase, _ = os.path.splitext(filename)
|
|
toml_path = os.path.join(dict_dir, filename)
|
|
# tab_path = os.path.join(dict_dir, filebase + ".tab")
|
|
|
|
# Parse the .toml file
|
|
metadata = parse_toml_file(toml_path)
|
|
|
|
# Generate the download URL for the main file
|
|
# main_download_url = f"https://github.com/catusf/tudien/releases/tag/{TAG_DOWNLOAD}/all-kindle.zip"
|
|
|
|
# Get the additional downloadable files
|
|
download_urls = get_downloadable_files(filebase, DOWNLOAD_TAG, output_dir)
|
|
|
|
for ext in SUPPORTED_EXTENSIONS:
|
|
if "pleco.zip" in ext and not filebase in pleco_data:
|
|
continue
|
|
|
|
needed_files.append(os.path.join(output_dir, filebase + "." + ext))
|
|
|
|
# Get full language names in Vietnamese
|
|
source_full_name = langcodes.Language.get(metadata["Source"]).display_name("vi")
|
|
# language_names.get(metadata['Source'], f"Unknown ({metadata['Source']})")
|
|
target_full_name = langcodes.Language.get(metadata["Target"]).display_name("vi")
|
|
# language_names.get(metadata['Target'], f"Unknown ({metadata['Target']})")
|
|
|
|
# Append the data to the list
|
|
data.append(
|
|
{
|
|
"Number": len(data) + 1, # Add numbering
|
|
"Name": metadata["Name"],
|
|
"Description": metadata["Description"],
|
|
"Source": f"{source_full_name} ({metadata['Source']})", # Full language name in Vietnamese
|
|
"Target": f"{target_full_name} ({metadata['Target']})", # Full language name in Vietnamese
|
|
"Owner_Editor": metadata["Owner_Editor"],
|
|
"Version": metadata["Version"],
|
|
"Num_entries": metadata["Num_entries"], # count_lines_in_tab(tab_path)
|
|
"Download": download_urls,
|
|
}
|
|
)
|
|
|
|
num_dict_found += 1
|
|
|
|
# for ext in SUPPORTED_EXTENSIONS:
|
|
# item = SUPPORTED_EXTENSIONS[ext]
|
|
# needed_files.append(os.path.join(output_dir, f"all-{item['dir']}.zip"))
|
|
|
|
# Save the list of dictionaries as a JSON file
|
|
json_path = os.path.join(dict_dir, "dict_summary.json")
|
|
with open(json_path, "w", encoding="utf-8") as json_file:
|
|
json.dump(data, json_file, ensure_ascii=False, indent=4)
|
|
|
|
print(f"Data file writtend to '{json_path}'.")
|
|
|
|
existing_files = sorted(glob.glob(os.path.join(output_dir, "*.*")))
|
|
|
|
missing_files = sorted(set(needed_files) - set(existing_files))
|
|
|
|
print("JSON file 'dict_summary.json' has been generated.")
|
|
|
|
files_status = "# Status report\n\n"
|
|
files_status += "## Counts\n\n"
|
|
files_per_format = len(SUPPORTED_EXTENSIONS)
|
|
existing_dicts = (len(existing_files) - files_per_format) / files_per_format
|
|
missing_dicts = len(missing_files) / files_per_format
|
|
|
|
if existing_dicts < 0:
|
|
existing_dicts = 0
|
|
missing_dicts -= 1
|
|
|
|
mismatched_dicts = num_dict_found - (existing_dicts + missing_dicts)
|
|
|
|
assert num_dict_found == len(data)
|
|
# assert files_per_format * (num_dict_found + 1) == len(needed_files)
|
|
|
|
files_status += f"- There are **{len(data)}** dict files.\n\n"
|
|
files_status += f"- Total NEEDED files: **{len(needed_files)}**\n\n"
|
|
files_status += f"- Total GENERATED files: **{len(existing_files)}** "
|
|
files_status += f"- or **{existing_dicts:.1f}** dictionary sets. "
|
|
|
|
normal = False
|
|
if len(missing_files) or len(existing_files) % files_per_format != 0:
|
|
files_status += "ABNORMAL NUMBER of files. Some dict has **missing format(s)**. Check missing files list for details.\n\n"
|
|
else:
|
|
files_status += "The number of files looks NORMAL.\n\n"
|
|
normal = True
|
|
|
|
files_status += f"- Total MISSING files: {len(missing_files)}** "
|
|
files_status += f"(or **{missing_dicts:.1f}** dictionaries which is {'CORRECT' if not missing_dicts and mismatched_dicts == 0 else 'IN-CORRECT'})\n\n"
|
|
|
|
files_status_details = ""
|
|
|
|
if not normal:
|
|
files_status_details += "# Errors\n"
|
|
|
|
files_status_details += f"## Missing files list\n\n"
|
|
for item in missing_files:
|
|
files_status_details += f"\t{item}\n"
|
|
|
|
if showfull:
|
|
print(f"showfull {showfull}")
|
|
# print(f"{files_status_details}")
|
|
# print(files_status)
|
|
|
|
return data, files_status, files_status_details
|
|
|
|
|
|
def generate_markdown_table(data, files_status, files_status_details, extensions, columns):
|
|
"""Generate a markdown table from the data."""
|
|
print(f"Generating report for {len(data)} dictionaries for {extensions}")
|
|
|
|
# Sort the data by "Source" then by "Name"
|
|
data = sorted(data, key=lambda x: (x["Source"], x["Name"]))
|
|
|
|
types = [SUPPORTED_EXTENSIONS[ext]["name"] for ext in extensions]
|
|
header = "| Number | Name | " # " Description | Source | Target | Owner/Editor | Num_entries | " + " | ".join(types)]
|
|
seperator = "| --- | --- | " # " --- | --- | --- | --- | --- |" + " --- |" * len(extensions)]
|
|
|
|
if "Description" in columns:
|
|
header += "Description |"
|
|
seperator += " --- |"
|
|
|
|
if "Source" in columns:
|
|
header += "Source |"
|
|
seperator += " --- |"
|
|
|
|
if "Target" in columns:
|
|
header += "Target |"
|
|
seperator += " --- |"
|
|
|
|
if "Owner_Editor" in columns:
|
|
header += "Owner_Editor |"
|
|
seperator += " --- |"
|
|
|
|
if "Version" in columns:
|
|
header += "Version |"
|
|
seperator += " --- |"
|
|
|
|
if "Num_entries" in columns:
|
|
header += "# Num_entries |"
|
|
seperator += " ---: |" # Right align number of definitions
|
|
|
|
header += " | ".join(types) + " |"
|
|
seperator += " --- |" * len(extensions)
|
|
|
|
header += " Number |"
|
|
seperator += " --- |"
|
|
|
|
markdown = [header, seperator]
|
|
|
|
for num, entry in enumerate(data, start=1):
|
|
download_links = " | ".join([f"[{ext}]({entry['Download'][ext]})" if entry['Download'][ext] else "N/A" for ext in extensions])
|
|
|
|
line = f"| {num} | {entry['Name']} | "
|
|
|
|
if "Description" in columns:
|
|
line += f"{entry['Description']} |"
|
|
|
|
if "Source" in columns:
|
|
line += f"{entry['Source']} |"
|
|
|
|
if "Target" in columns:
|
|
line += f"{entry['Target']} |"
|
|
|
|
if "Owner_Editor" in columns:
|
|
line += f"{entry['Owner_Editor']} |"
|
|
|
|
if "Version" in columns:
|
|
line += f"{entry['Version']} |"
|
|
|
|
if "Num_entries" in columns:
|
|
n = entry['Num_entries']
|
|
line += f"{n:,} |"
|
|
|
|
line += f" {download_links} |"
|
|
|
|
line += f" {num} |"
|
|
|
|
markdown.append(line)
|
|
|
|
markdown.insert(0, files_status_details)
|
|
markdown.insert(0, files_status)
|
|
|
|
return "\n".join(markdown)
|
|
|
|
|
|
def main():
|
|
"""Main function to parse arguments and run the processes.""" # noqa: D202, D401
|
|
|
|
# Parse command-line arguments
|
|
parser = argparse.ArgumentParser(description="Generate a dictionary summary.")
|
|
parser.add_argument("-d", "--dict_dir", default="dict", help="The directory containing the dictionary files (default is 'dict').")
|
|
parser.add_argument("-f", "--outfile", default="dict_summary.md", help="The output report file name (default is 'dict_summary.md').")
|
|
parser.add_argument("-o", "--output_dir", default="output", help="The output dir for all the dict results.")
|
|
parser.add_argument("-e", "--extensions", default=None, help="The extensions that need included in the report. None means all.")
|
|
parser.add_argument("-c", "--columns", default=None, help="The columns that will be kept (Other than the download links).")
|
|
parser.add_argument("-r", "--read_only", choices=["yes", "no"], default="no", required=False, help="Read data or create it.")
|
|
parser.add_argument("-s", "--showfull", default=True, help="Shows full details of report in the console.")
|
|
|
|
args = parser.parse_args()
|
|
|
|
print(args)
|
|
|
|
# Generate the summary data and save it as a JSON file
|
|
ext_str = args.extensions
|
|
col_str = args.columns
|
|
read_only = args.read_only
|
|
dict_dir = args.dict_dir
|
|
outfile = args.outfile
|
|
output_dir = args.output_dir
|
|
showfull = args.showfull
|
|
|
|
extensions = list(SUPPORTED_EXTENSIONS.keys()) if not ext_str else [item.strip() for item in ext_str.split(",")]
|
|
columns = list(COLUMNS.keys()) if not col_str else [item.strip() for item in col_str.split(",")]
|
|
|
|
files_status = ""
|
|
files_status_details = ""
|
|
if read_only == "no":
|
|
data, files_status, files_status_details = generate_summary(dict_dir, output_dir, showfull=showfull)
|
|
else:
|
|
json_path = os.path.join(dict_dir, "dict_summary.json")
|
|
with open(json_path, "r", encoding="utf-8") as json_file:
|
|
data = json.load(json_file)
|
|
|
|
# Generate the markdown table from the JSON data
|
|
markdown_table = generate_markdown_table(data, files_status, files_status_details, extensions, columns)
|
|
|
|
# Save the markdown table to a .md file
|
|
markdown_file = os.path.join(dict_dir, outfile)
|
|
with open(markdown_file, "w", encoding="utf-8") as file:
|
|
file.write(markdown_table)
|
|
|
|
if showfull:
|
|
print(f"{showfull} Data file written to: {markdown_file}")
|
|
|
|
print(f"Summary markdown file '{outfile}' has been generated.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|