chinese-xinhua/scripts/clean.ipynb

Clean Data (Duplicate Removal)

In [ ]:

import json

In [ ]:

with open("/home/pwxcoo/projects/chinese-xinhua/archived/idiom.json") as fp:
    idioms = json.load(fp)

len(idioms)

In [ ]:

idioms[:5]

In [ ]:

unique_idioms = set(idiom["word"] for idiom in idioms)
unique_idioms

In [ ]:

len(unique_idioms)

In [ ]:

check = dict()
for idiom in idioms:
    if idiom["word"] in check and str(idiom) > str(check[idiom["word"]]):
#         print(idiom)
#         print(check[idiom["word"]])
        check[idiom["word"]] = idiom
    else:
        check[idiom["word"]] = idiom

In [ ]:

with open("/home/pwxcoo/projects/chinese-xinhua/data/idiom-clean.json", "w+", encoding="utf-8") as fp:
    json.dump(list(check.values()), fp, ensure_ascii=False)

In [ ]: