#!/usr/bin/env python3 """Script to build all dictionaries with all formats Usage: ./bin/convert_all.py --input_folder=./dict --output_folder=./output --extension=tab """ import argparse import glob import os import re import shlex import shutil import subprocess from fileinput import filename from pathlib import Path from dict_summary import parse_toml_file from iso_language_codes import language_name # List of dictionary formats to generate using pyglossary DICT_FORMATS = [ # DictType, foldername, file ending, need zip # DictType is the pyglossary write-format name ("Yomichan", "yomitan", "yomitan.zip", False), ("Epub2", "epub", "epub", False), ("Kobo", "kobo", "kobo.zip", False), ("Stardict", "stardict", "ifo", True), ("DictOrg", "dictd", "index", True), ("Aard2Slob", "aard", "slob", False), ] # List of directory formats and their file patterns DIR_FORMATS = [ # (Dir, File pattern) ("stardict", "*.stardict.zip"), ("epub", "*.epub"), ("kobo", "*.kobo.zip"), ("lingvo", "*.dsl.dz"), ("kindle", "*.mobi"), ("dictd", "*.dictd.zip"), ("yomitan", "*.yomitan.zip"), ("mdict", "*.mdx"), ("pleco", "*.pleco.zip"), ("aard", "*.slob"), ("dic", "*.dic"), # ("xdxf", "*.xdxf.zip"), ] def execute_shell(cmd_line, message="", printout=True): """ Executes a shell command and handles any errors that occur during execution. Parameters: cmd_line (str): The command line string to be executed. message (str): An optional message to include in error output if the command fails. printout (bool): If True, prints the command line before execution. Returns: bool: True if the command executes successfully, False if an error occurs. """ # noqa: D401 try: if printout: print(cmd_line) # subprocess.run(shlex.split(cmd_line), check=True) subprocess.call(cmd_line, shell=True) return True except subprocess.CalledProcessError as e: if message: print(f"Error {message}: {e}") else: print(f"Error executing shell: {e}") return False DEBUG_FLAG = False def escape_forbidden_chars(text, forbidden_chars=r" (){}[]$*?^|<>\\"): """ Escapes forbidden characters in a given text. Args: text (str): The input string to process. forbidden_chars (str): A string containing characters to escape (default: common special chars). Returns: str: The text with forbidden characters escaped. """ # Create a regex pattern to match any of the forbidden characters pattern = f"[{re.escape(forbidden_chars)}]" # Escape each match by prefixing it with a backslash escaped_text = re.sub(pattern, r"\\\g<0>", text) return escaped_text # Example usage def gen_mdict_target(filepath, filebase, output_folder, dataName, dataDescription): """ Generate MDict source files (*.title.html, *.description.html, *.txt) and build an MDX dictionary. This function creates the necessary metadata and definition files for building an MDict (.mdx) file from a tab-separated input text file. Each valid line in the input must contain exactly one tab: `\t`. The function then calls the `mdict` command-line tool to produce the MDX file. Parameters: filepath (str): Path to the input text file containing dictionary entries. Each line must have one headword and one definition separated by a tab. filebase (str): Base filename (without extension) used for generated output files. output_folder (str): Directory where all output files will be created. dataName (str): Title of the dictionary, written to `.title.html`. dataDescription (str): Description of the dictionary, written to `.description.html`. Output Files Generated (in output_folder): .title.html Contains the dictionary title. .description.html Contains the dictionary description. .txt Contains formatted entries for MDict (headword + definition). .mdx Final MDict dictionary output (built by `mdict` CLI). Behavior: • Skips empty lines. • Skips lines without exactly one tab, printing a warning for each skipped line. • Writes entries to the .txt file in the MDict expected format: headword definition • Executes the `mdict` tool to build the `.mdx` file. Returns: Result of the `execute_shell()` call that runs the MDX generation command. """ title_filepath = os.path.join(output_folder, filebase + ".title.html") with open(title_filepath, "w", encoding="utf-8") as file: file.write(dataName) desc_filepath = os.path.join(output_folder, filebase + ".description.html") with open(desc_filepath, "w", encoding="utf-8") as file: file.write(dataDescription) def_filepath = os.path.join(output_folder, filebase + ".txt") dict_filepath = os.path.join(output_folder, filebase + ".mdx") print(f"{filepath} - {output_folder} - {dict_filepath}") with open(def_filepath, "w", encoding="utf-8") as outfile: with open(filepath, "r", encoding="utf-8") as file: for line in file: line = line.strip() # remove trailing newline if not line: continue if line.count("\t") != 1: print(f"Warning: line in {filepath} has no tab or multiple tabs, skipping: {line!r}") continue # assert(line.count("\t") == 1) headword, definition = line.split("\t") # split by tab outfile.write(f"{headword}\n{definition}\n\n") cmd_line = ( f"mdict --title {title_filepath} --description {desc_filepath} -a {def_filepath} {dict_filepath}" # noqa: E501 ) return execute_shell(cmd_line=cmd_line, message=f"generating MDict MDX") import re def split_long_text_file( input_path, output_path=None, max_len=800 ): item_pattern = re.compile(r'(?=\b\d+\. )') ascii_word = re.compile(r'[A-Za-z0-9_]') zh_punct = '。；，' output_lines = [] def split_by_length(text): chunks = [] while len(text) > max_len: cut = -1 # 1. Try space (English) cut = text.rfind(' ', 0, max_len) # 2. Try Chinese punctuation if cut == -1: for p in zh_punct: pos = text.rfind(p, 0, max_len) if pos > cut: cut = pos + 1 # include punctuation # 3. Smart fallback if cut == -1: cut = max_len # Avoid breaking ASCII words / tags if ( cut < len(text) and ascii_word.match(text[cut - 1]) and ascii_word.match(text[cut]) ): forward = re.search(r'\s', text[cut:cut + 100]) if forward: cut += forward.start() chunks.append(text[:cut].rstrip()) text = text[cut:].lstrip() if text: chunks.append(text) return chunks with open(input_path, 'r', encoding='utf-8') as f: for line in f: line = line.strip() if not line: continue items = item_pattern.split(line) for item in items: item = item.strip() if not item: continue output_lines.extend(split_by_length(item)) if output_path: with open(output_path, 'w', encoding='utf-8') as f: f.write('\n'.join(output_lines)) return output_lines def main() -> None: """Main entry point for converting dictionary files into multiple formats. This function processes dictionary files and their metadata to convert them into various dictionary formats including StarDict, EPUB, Kobo, Lingvo, Kindle, DictD, Yomitan, and MDict. Args: None, but accepts command line arguments: -i, --input_folder: Input folder containing dictionary files (default: 'dict') -o, --output_folder: Output folder for converted dictionaries (default: 'output') -e, --extension: File extension for input dictionary files (default: 'tab') -m, --metadata: File extension for input metadata files (default: 'toml') -f, --filter: Optional comma-separated list of dictionary keys to filter Returns: None The function performs the following operations: 1. Parses command line arguments 2. Searches for dictionary and metadata files 3. Creates necessary output directories 4. Converts dictionaries to multiple formats: - Kindle (MOBI) - Pleco (TXT) - Lingvo (DSL) - MDict (MDX) - Other formats specified in DICT_FORMATS 5. Creates ZIP archives of converted dictionaries Note: Requires various external dependencies and shell commands for dictionary conversion. Temporary files are cleaned up during processing to save space. """ parser = argparse.ArgumentParser(description="Convert all dictionaries in a folder") parser.add_argument("-i", "--input_folder", default="dict", help="Input folder containing .tsv and .dfo files") parser.add_argument("-o", "--output_folder", default="output", help="Output folder containing dictionary files") parser.add_argument("-e", "--extension", default="tab", help="Filename extention for input dictionary files. Default is .tab") parser.add_argument("-m", "--metadata", default="toml", help="Filename extention for input metadata for dictionary. Default is .dfo") parser.add_argument("-f", "--filter", help="Filter only dictionary entries with matching keys (seperated by comma)") args = parser.parse_args() input_folder = Path(escape_forbidden_chars(args.input_folder)) output_folder = Path(escape_forbidden_chars(args.output_folder)) extension = args.extension metadata = args.metadata dict_filters = args.filter.split(",") if args.filter is not None else [] print(f"Arguments: {args}") metafilelist = [] datafilelist = [] metafilelist, datafilelist = search_data_files(input_folder, extension, metadata, dict_filters) # dirs = ["stardict", "epub", "kobo", "lingvo", "kindle", "dictd", "yomitan", "mdict"] output_kindle = output_folder / "kindle" output_mdict = output_folder / "mdict" dict_kindle = input_folder / "kindle" # cmd_line = f"mkdir -p {input_folder}/kindle" # execute_shell(cmd_line=cmd_line, message="Creating directory") # subprocess.call(f'rm -r {output_folder}/*', shell=True) dict_kindle.mkdir(parents=True, exist_ok=True) # cmd_line = f"rm -r {output_folder}/*" # execute_shell(cmd_line=cmd_line, message="Remove existing file in {output_folder}") shutil.rmtree(output_folder, ignore_errors=True) # cmd_line = f"rm -r {input_folder}/kindle/*" # execute_shell(cmd_line=cmd_line, message="Remove existing file in kindle {input_folder}/kindle") shutil.rmtree(output_kindle, ignore_errors=True) for dir, _ in DIR_FORMATS: sub_path = Path(output_folder / dir) sub_path.mkdir(parents=True, exist_ok=True) # cmd_line = f"mkdir -p {output_folder}/{dir}" # execute_shell(cmd_line=cmd_line, message="Creating directory") # subprocess.call(f'mkdir -p {output_folder}/{dir}', shell=True) # use_only_these = {'Tu-dien-ThienChuu-TranVanChanh'} for filepath, datafile in zip(metafilelist, datafilelist): _, filename = os.path.split(filepath) filebase, fileext = os.path.splitext(filename) data = parse_toml_file(filepath) if not data: continue # Add quote to wrap long filename/path #datafile = datafile.replace(" ", "\\ ") # safe = shlex.quote(str(datafile)) dataCreator = data["Owner_Editor"].replace(" ", "\\ ") if not dataCreator: dataCreator = "Panthera Tigris".replace(" ", "\\ ") dataTarget = data["Target"] dataSource = data["Source"] dataDescription = data["Description"] dataFullSource = language_name(data["Source"]) dataFullTarget = language_name(data["Target"]) dataName = escape_forbidden_chars(data["Name"]) INFLECTION_DIR = "./bin/inflections" INFLECTION_NONE = "inflections-none.tab" # Generare HTML file for Kindle dictionary if "Inflections" in data and data["Inflections"]: inflections = f'"{INFLECTION_DIR}/{data["Inflections"]}"' else: inflections = f"{INFLECTION_DIR}/{INFLECTION_NONE}" # Generate other dictionary formats using PyGlossary for write_format, folder, ext, needzip in DICT_FORMATS: _, filename = os.path.split(filepath) filebase, _ = os.path.splitext(filename) build_dict_many(output_folder, ext, datafile, filebase, dataTarget, dataSource, dataName, write_format, folder, needzip) # Must run after dictd was created build_dict_pocketbook(output_folder, filebase) # cmd_line = f"ls -llR {output_folder}" # execute_shell(cmd_line=cmd_line, message=f"Listing all files in {output_folder}") build_dict_mobi(input_folder, output_folder, datafile, filebase, inflections, dataCreator, dataTarget, dataSource, dataName) build_dict_pleco_txt(input_folder, output_folder, filebase) build_dict_lingvo(input_folder, output_folder, datafile, filebase, dataCreator, dataFullTarget, dataFullSource, dataName) build_dict_mdict(output_folder, datafile, filebase, dataDescription, dataName) # if DEBUG_FLAG: # continue # Deletes src data file to save space delete_file(f"{filebase}.txt", input_folder) delete_file(f"{filebase}.{extension}", input_folder) # cmd_line = f"rm {input_folder}/{filebase}.txt" # Still keeps .tab file for counting lines later # execute_shell(cmd_line=cmd_line, message=f"Removes {filebase} input files to save space") # cmd_line = f"ls -llR {output_folder}" # execute_shell(cmd_line=cmd_line, message=f"Listing all files in {output_folder}") for dir, format in DIR_FORMATS: cmd_line = f"zip -9 -j {output_folder}/all-{dir}.zip {output_folder}/{format}" execute_shell(cmd_line=cmd_line, message=f"Zipping all {dir}-format dicts in {output_folder}") def delete_file(filename, input_folder=""): """ Delete a single file if it exists. Parameters ---------- filename : str Name of the file to delete. input_folder : str, optional Path to the folder containing the file (default is current directory). Notes ----- If the file does not exist, the function does nothing. """ file_path = Path(input_folder) / filename if file_path.exists(): file_path.unlink() def delete_file_pattern(filename, pattern, dir_path=""): """ Delete all files in a directory that match a filename pattern. Parameters ---------- filename : str Base name of the files to match. pattern : str Pattern appended to filename (e.g., "*.txt" or "_backup.*"). dir_path : str, optional Directory to search for matching files (default is current directory). Notes ----- Only files are deleted. Subdirectories are ignored. """ dir = Path(dir_path) for f in dir.glob(f"{filename}{pattern}"): if f.is_file(): f.unlink() def build_dict_many(output_folder, extension, datafile, filebase, dataTarget, dataSource, dataName, write_format, folder, needzip): """ Generate a dictionary file from a tab-delimited data source using pyglossary, optionally compress the result into a ZIP archive, and move or clean up intermediate files. This function builds command lines for pyglossary and common shell utilities (zip, rm, mv) and delegates execution to `execute_shell`. It is intended to create a converted dictionary file at: //. or, when `needzip` is True, to create a ZIP at: /..zip Parameters ---------- output_folder : str Path to the base output directory where resulting files or ZIPs will be placed. extension : str File extension for the generated file (used in the output filename). datafile : str Path to the input data file (expected Tabfile format) to be converted. filebase : str Base name (without extension) for the generated file or ZIP. dataTarget : str Target language code passed to pyglossary via --target-lang. dataSource : str Source language code passed to pyglossary via --source-lang. dataName : str Dictionary name passed to pyglossary via --name. pyglossary : str Path to the pyglossary executable or a placeholder parameter. Note: the current implementation does not use this parameter directly when building the command line (pyglossary is invoked by literal name). write_format : str Output format name passed to pyglossary via --write-format. folder : str Subfolder inside `output_folder` where the non-zipped output is placed. needzip : bool If True, compress the generated output files into a ZIP and remove the temporary generated files; if False, move the generated file(s) into `output_folder`. Returns ------- None Side effects ------------ - Executes shell commands via `execute_shell`: - Runs pyglossary to convert the input file to the requested format. - If `needzip` is True: * Creates a ZIP containing the generated files and deletes the temporary files produced in the subfolder. Otherwise: * Moves the produced file(s) from the subfolder into `output_folder`. - Creates, moves, and deletes files on the filesystem. Errors and exceptions --------------------- - Any exception or error produced by `execute_shell` (or underlying shell commands) will propagate to the caller. - The function currently interpolates many values directly into shell command strings. For safety, ensure that inputs are trusted or properly sanitized to avoid shell injection vulnerabilities. Only `out_path` is quoted with `shlex.quote` in the current implementation. Notes ----- - The function assumes `pyglossary` and standard Unix utilities (`zip`, `rm`, `mv`) are available in PATH, or that `execute_shell` knows how to locate them. - `pyglossary` parameter is accepted but not inserted into constructed command lines in the current implementation; update the command construction if you need to use a non-default pyglossary executable. """ out_path = os.path.join(output_folder, folder, f"{filebase}.{extension}") cmd_line = ( f"pyglossary --ui=none --read-format=Tabfile --write-format={write_format} " f"--source-lang={dataSource} --target-lang={dataTarget} --name={dataName} {datafile} {shlex.quote(out_path)}" ) execute_shell(cmd_line=cmd_line, message=f"generating {write_format}") if needzip: out_path = os.path.join(output_folder, f"{folder}/{filebase}.*") zip_path = os.path.join(output_folder, f"{filebase}.{folder}.zip") cmd_line = f"zip -j -9 {zip_path} {out_path}" execute_shell(cmd_line=cmd_line, message=f"creating zip file for {write_format} in {output_folder}") if folder not in ("dictd"): # keep dictd files for Pocketbook conversion out_dir = Path(output_folder) / folder for f in out_dir.glob(f"{filebase}.*"): if f.is_file(): f.unlink() # cmd_line = f"rm {out_path}" # execute_shell(cmd_line=cmd_line, message=f"Remove temp file for {filebase} in {output_folder}") else: # cmd_line = f"mv {out_path} {output_folder}" # execute_shell(cmd_line=cmd_line, message=f"moving output {out_path} to output folder {output_folder}") Path(out_path).replace(Path(output_folder) / f"{filebase}.{extension}") def build_dict_mobi(input_folder, output_folder, datafile, filebase, inflections, dataCreator, dataTarget, dataSource, dataName): """ Build a Kindle-compatible .mobi dictionary from intermediate data files. This function orchestrates the creation of a Kindle dictionary by: 1. Running a conversion script (tab2opf.py) to produce an OPF+HTML set in a temporary "kindle" subdirectory under the provided input folder. 2. Invoking the mobigen.exe tool (via wine) to generate a .mobi file from the generated OPF file. 3. Moving the resulting .mobi file to the specified output folder. 4. Cleaning up generated HTML files for the processed file base. Note: this function executes external commands and relies on several external binaries and scripts being present and executable (python ./bin/tab2opf.py, wine, ./bin/mobigen/mobigen.exe). It prints command lines it runs and uses subprocess and an execute_shell helper for execution. Args: input_folder (str): Path to the input folder containing source materials. A "kindle" subdirectory under this folder will be used as the temporary output directory for OPF/HTML files. output_folder (str): Destination directory where the final .mobi file will be moved. datafile (str): Path to the source data file passed as the final argument to tab2opf.py. filebase (str): Base filename (without extension) used to name the generated OPF/HTML/mobi artefacts. inflections (str): Value passed to tab2opf.py's --inflection option that controls whether/which inflections to include. dataCreator (str): Creator/author name passed to tab2opf.py (also used as publisher in the current command). dataTarget (str): Target language/code passed to tab2opf.py’s --target option. dataSource (str): Source language/code passed to tab2opf.py’s --source option. dataName (str): Title/name passed to tab2opf.py’s --title option and used as metadata in the generated OPF. Returns: None Side effects: - Creates (and later removes) HTML files in {input_folder}/kindle. - Creates an OPF file and a .mobi file for the given filebase. - Moves the .mobi file into the provided output_folder. - Prints executed command lines to stdout. Errors and exceptions: - External command failures may raise subprocess exceptions or fail silently depending on how subprocess.run/subprocess.call are invoked. The function also depends on the behavior of execute_shell for cleanup; any errors raised by that helper will propagate. - Because command lines are assembled from input parameters, passing untrusted input may lead to shell injection risks (note that some calls use shlex.split while others invoke a shell). Sanitize or validate inputs before calling this function in untrusted contexts. Example: build_dict_mobi( input_folder="/path/to/workspace", output_folder="/path/to/output", datafile="/path/to/data.tsv", filebase="my_dictionary", inflections="yes", dataCreator="Example Author", dataTarget="en", dataSource="la", dataName="Latin–English Dictionary" ) """ html_dir = f"kindle" html_out_dir = f"{input_folder}/{html_dir}" cmd_line = f"python ./bin/tab2opf.py --title={dataName} --source={dataSource} --target={dataTarget} --inflection={inflections} --outdir={html_out_dir} --creator={dataCreator} --publisher={dataCreator} {datafile}" # noqa: E501 print(cmd_line) subprocess.run(shlex.split(cmd_line)) # Generate .mobi dictionary from opf+html file out_path = f"{html_out_dir}/{filebase}.opf".replace(" ", "\\ ") cmd_line = f"wine ./bin/mobigen/mobigen.exe -unicode -s0 {out_path}" print(cmd_line) subprocess.run(shlex.split(cmd_line)) cmd_line = f"mv {html_out_dir}/{filebase}.mobi {output_folder}/" print(cmd_line) subprocess.call(cmd_line, shell=True) # cmd_line = f"rm {htmlOutDir}/{filebase}*.html" # execute_shell(cmd_line=cmd_line, message=f"Removes html files for {filebase}") delete_file_pattern(filebase, "*.html", html_out_dir) def build_dict_pocketbook(output_folder, filebase): """ Builds an Pocketbook from dictd file. This MUST RUN AFTER dictd was created Args: output_folder (str): Directory path where the output dictionary will be saved datafile (str): Path to input data file filebase (str): Base filename to use for generated dictionary dataDescription (str): Description of the dictionary data dataName (str): Name of the dictionary The function: 1. Generates an XDXF datafile using makedict tool 2. Use Pocketbook LanguageFilesPocketbookConverter to convert XDXF to Pocketbook .dic format 3. Renames output file Requires: - gen_mdict_target() function to generate the dictionary - execute_shell() function to run shell commands """ # Generare Pocketbook dictionary out_dictdir = os.path.join(output_folder, "dic") out_xdxfdir = os.path.join(output_folder, "xdxf") # cmd_line = f"mkdir {out_dictdir}" # print(cmd_line) # subprocess.run(shlex.split(cmd_line)) dictd_base = f"{output_folder}/dictd/{filebase}" # Generare xdxf intermediate dictionary format cmd_line = f"makedict -i dictd -o xdxf {dictd_base}.index --work-dir {out_dictdir}" print(cmd_line) subprocess.run(shlex.split(cmd_line)) cmd_line = f"rm {dictd_base}.index {dictd_base}.dict" print(cmd_line) subprocess.run(shlex.split(cmd_line)) converter = "wine LanguageFilesPocketbookConverter/converter.exe" lang_data = "./LanguageFilesPocketbookConverter/en/" old_xdxf_file = f"{out_dictdir}/{filebase}/dict.xdxf" new_xdxf_file = f"{out_dictdir}/{filebase}.xdxf" old_dic_file = f"{out_dictdir}/{filebase}.dic" cmd_line = f"mv {old_xdxf_file} {new_xdxf_file}" print(cmd_line) subprocess.run(shlex.split(cmd_line)) # Split the lines by item number, to overcome a limitation of `converter` split_long_text_file(new_xdxf_file, new_xdxf_file) cmd_line = f"{converter} {new_xdxf_file} {lang_data}" print(cmd_line) subprocess.run(shlex.split(cmd_line)) cmd_line = f"rm {new_xdxf_file}" print(cmd_line) subprocess.run(shlex.split(cmd_line)) cmd_line = f"mv {old_dic_file} {output_folder}/" print(cmd_line) subprocess.run(shlex.split(cmd_line)) # cmd_line = f"mv {old_dic_file} {new_dic_file}" # print(cmd_line) # subprocess.run(shlex.split(cmd_line)) # cmd_line = f"zip -j -m -9 {output_folder}/{filebase}.xdxf.zip {out_xdxfdir}/{filebase}.xdxf" # execute_shell(cmd_line=cmd_line, message=f"Making Pleco dict file for {filebase}") # cmd_line = f"rm {output_folder}/mdict/{filebase}*" # execute_shell(cmd_line=cmd_line, message=f"Removes temp mdict files in {output_folder}/mdict") # delete_file_pattern(filebase, "*", f"{output_folder}/mdict/") def build_dict_mdict(output_folder, datafile, filebase, dataDescription, dataName): """ Builds an MDict from a data file. Args: output_folder (str): Directory path where the output dictionary will be saved datafile (str): Path to input data file filebase (str): Base filename to use for generated dictionary dataDescription (str): Description of the dictionary data dataName (str): Name of the dictionary The function: 1. Generates an MDict in a temporary folder 2. Moves .mdx files to the output folder 3. Cleans up temporary files Requires: - gen_mdict_target() function to generate the dictionary - execute_shell() function to run shell commands """ # Generare Mdict dictionary out_dictdir = os.path.join(output_folder, "mdict") gen_mdict_target(datafile, filebase, out_dictdir, dataName, dataDescription) cmd_line = f"mv {out_dictdir}/*.mdx {output_folder}/" execute_shell(cmd_line=cmd_line, message=f"moving output {output_folder} to output folder {output_folder}") # cmd_line = f"rm {output_folder}/mdict/{filebase}*" # execute_shell(cmd_line=cmd_line, message=f"Removes temp mdict files in {output_folder}/mdict") delete_file_pattern(filebase, "*", f"{output_folder}/mdict/") def build_dict_pleco_txt(input_folder, output_folder, filebase): """ Create a Pleco-compatible zipped dictionary file (.pleco.zip) from a plaintext dictionary file located in input_folder. This function looks for a file named "{filebase}.txt" in input_folder. If that file exists, it runs a shell zip command to create a zip archive named "{filebase}.pleco.zip" in output_folder containing the original .txt file. The zip is created with the -j option (junk the path) so the archive contains only the .txt file, not its parent directories. Parameters ---------- input_folder : str Path to the folder that contains the source "{filebase}.txt" file. output_folder : str Path to the folder where the resulting "{filebase}.pleco.zip" archive should be written. filebase : str Base filename (without extension) used to compose the source ".txt" and destination ".pleco.zip" filenames. Returns ------- None Side effects ------------ - If "{input_folder}/{filebase}.txt" exists, a shell command is executed via execute_shell to create the zip archive at "{output_folder}/{filebase}.pleco.zip". - If the source .txt file does not exist, the function does nothing. Exceptions ---------- Any exceptions raised by execute_shell or underlying system calls (e.g., permission errors, missing external zip utility) will propagate to the caller. Example ------- build_dict_pleco_txt("/path/to/input", "/path/to/output", "mydict") # If "/path/to/input/mydict.txt" exists, creates "/path/to/output/mydict.pleco.zip" """ # Zip to create Pleco dictionary pleco_dict_file = f"{input_folder}/{filebase}.txt" if os.path.exists(pleco_dict_file): cmd_line = f"zip -j -m -9 {output_folder}/{filebase}.pleco.zip {input_folder}/{filebase}.txt" execute_shell(cmd_line=cmd_line, message=f"Making Pleco dict file for {pleco_dict_file}") def build_dict_lingvo(input_folder, output_folder, datafile, filebase, dataCreator, dataFullTarget, dataFullSource, dataName): """ Generate a Lingvo (.dsl) dictionary by invoking an external tab2dsl Ruby tool and move the generated archive to the output folder. This function constructs and runs shell commands to: 1. Call the Ruby script ./dsl-tools/tab2dsl/tab2dsl.rb to convert a tabular dictionary file into a Lingvo DSL file. 2. Move the resulting compressed output (.dz) from the temporary output location into the provided output folder. Notes: - The temporary DSL output path is constructed as "/lingvo/.dsl" with spaces escaped. - The function relies on an available `execute_shell` helper to run shell commands; any exceptions raised by that helper will propagate. - The Ruby script must exist at ./dsl-tools/tab2dsl/tab2dsl.rb and be executable in the running environment. - The parameter `dataCreator` is accepted but not used by this function. Parameters ---------- input_folder : str Path to the folder containing input resources (not directly used by the current implementation but kept for API consistency). output_folder : str Destination folder where the final .dz archive will be moved. datafile : str Path to the tab-separated dictionary file that will be converted by the tab2dsl tool. filebase : str Base name used for the generated DSL file (the function creates "lingvo/{filebase}.dsl" under output_folder). dataCreator : str Metadata field intended to represent the creator of the data. Note: currently not used by the function. dataFullTarget : str Target language code/name passed to the tab2dsl tool via --to-lang. dataFullSource : str Source language code/name passed to the tab2dsl tool via --from-lang. dataName : str Dictionary name passed to the tab2dsl tool via --dict-name. Returns ------- None The function performs side effects (file creation and movement) and does not return a value. Raises ------ Exception Propagates any exception raised by the underlying `execute_shell` calls (e.g., subprocess errors, OSError), indicating failure of the external tool invocation or filesystem operations. Examples -------- # Typical usage build_dict_lingvo( input_folder="/path/to/input", output_folder="/path/to/output", datafile="/path/to/input/data.tsv", filebase="my_dictionary", dataCreator="Example Creator", dataFullTarget="ru", dataFullSource="en", dataName="My Dictionary" """ # Generare Lingvo dictionary out_path = os.path.join(output_folder, f"lingvo/{filebase}.dsl").replace(" ", "\\ ") cmd_line = ( f"ruby ./dsl-tools/tab2dsl/tab2dsl.rb --from-lang {dataFullSource} --to-lang {dataFullTarget} --dict-name {dataName} --output {out_path} {datafile}" # noqa: E501 ) execute_shell(cmd_line=cmd_line, message=f"generating DSL/Longvo") cmd_line = f"mv {out_path}.dz {output_folder}" execute_shell(cmd_line=cmd_line, message=f"moving output {out_path} to output folder {output_folder}") def search_data_files(input_folder, extension, metadata, dict_filters): """ Search for matching metadata and data files in a specified directory. This function looks for pairs of metadata and dictionary data files in the input folder, optionally filtering the results based on specified dictionary names. Args: input_folder (str): Path to the directory containing the data files extension (str): File extension for the dictionary data files metadata (str): File extension for the metadata files dict_filters (list): Optional list of strings to filter dictionary names. Only dictionaries containing any of these strings in their names will be included. Returns: tuple: A pair of lists (metafilelist, datafilelist) containing paths to matching metadata and data files. For compressed files (.bz2), they will be decompressed and the path to the decompressed file will be returned. Notes: - The function handles both regular and bzip2 compressed (.bz2) data files - Files are matched based on their base names (without extensions) - When dict_filters is provided, only dictionaries with names containing any of the filter strings will be included - Compressed files are automatically decompressed during processing """ if input_folder: metafilelist = sorted(input_folder.glob(f"*.{metadata}"), reverse=True) datafilelist = sorted(input_folder.glob(f"*.{extension}"), reverse=True) zippdatafilelist = sorted(input_folder.glob(f"*.bz2"), reverse=True) meta_dict = {} data_dict = {} print(f"Len of metafilelist: {len(metafilelist)}") print(f"Len of datafilelist: {len(datafilelist)}") print(f"Len of compressed datafilelist: {len(zippdatafilelist)}") bothdatalist = datafilelist + zippdatafilelist # Lists of data files including compressed ones # Keep only pairs of metadata and dict data files meta_dict = {full_stem(filepath): filepath for filepath in metafilelist} data_dict = {full_stem(filepath): filepath for filepath in bothdatalist} common_keys = sorted(list(meta_dict.keys() & data_dict.keys())) print(common_keys) metafilelist.clear() datafilelist.clear() for key in common_keys: include_dict = False if dict_filters: for filter in dict_filters: if filter in key: include_dict = True break if not include_dict: print(f"Excluding this dictionary: {key}") continue metafilelist.append(meta_dict[key]) datafile = data_dict[key] # If datafile is a .bz2 if datafile.suffix == ".bz2": cmd_line = f'bzip2 -d "{str(datafile)}"' # Add -k to keep the original file execute_shell(cmd_line=cmd_line, message=f"bunzip data file") datafilelist.append(datafile.with_suffix("")) # Remove .bz2 suffix else: datafilelist.append(datafile) print(f"Len of checked datafilelist: {len(datafilelist)}") return metafilelist,datafilelist def full_stem(path): """ Return the filename without any extensions for files with multiple suffixes (e.g., .tar.gz). Files with a single suffix behave as path.stem Parameters ---------- path : str or Path The path to the file. Returns ------- str The filename with all suffixes removed. """ p = Path(path) name = p.name for _ in p.suffixes: name = Path(name).with_suffix("") return str(name) if __name__ == "__main__": """" Main entry point """ main()