Li_Feiyu/chinese-xinhua
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mirror: 初始快照

Browse Source
This commit is contained in:
Li_Feiyu committed 2026-10-02 22:47:01 +08:00
commit a8faa15291
16 files changed
+790012
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/README.md
+11
View File
@@ -0,0 +1,11 @@
## Scripts
这里是抓取整理数据的脚本们。
- [addAbbreviation.py](./addAbbreviation.py): 给成语添加缩写
- [clean.ipynb](./clean.ipynb): 去重重复成语
- [chengyu.py](./chengyu.py): 抓取成语
- [ci.py](./ci.py): 抓取词语
- [word.py](./word.py): 抓取汉字
- [xiehouyu.py](./xiehouyu.py): 抓取歇后语
scripts/addAbbreviation.py
+14
View File
@@ -0,0 +1,14 @@
import pandas as pd
chengyu = pd.read_json('chengyu.json')
replace = {'ā':'a', 'á':'a', 'ǎ':'a', 'à':'a', 'ō':'o', 'ó':'o', 'ǒ':'o', 'ò':'o',\
'ē':'e', 'é':'e', 'ě':'e', 'è':'e'}
def abbreviation(pinyin):
each = pinyin.split(' ')
return ''.join(list(map(lambda x: replace[x[:1]] if x[:1] in replace else x[:1], each)))
chengyu['abbreviation'] = chengyu['pinyin'].apply(abbreviation)
chengyu.to_json('idiom.json', force_ascii=False, orient='records')
scripts/chengyu.py
+54
View File
@@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
"""
author: pwxcoo
date: 2018-02-05
description: 抓取下载成语并保存
"""
import requests, json
from bs4 import BeautifulSoup
def downloader(url):
"""
下载成语并保存
"""
response = requests.get(url)
if response.status_code != 200:
print(f'{url} is failed!')
return
print(f'{url} is parsing')
html = BeautifulSoup(response.content.decode('gbk', errors='ignore'), "lxml")
table = html.find_all('table')[-2]
prefix = 'http://www.zd9999.com'
words = [prefix + a.get('href') for a in table.find_all('a')]
res = []
for i in range(0, len(words)):
response = requests.get(words[i])
print(f'{[words[i]]} is parsing')
if response.status_code != 200:
print(f'{words[i]} is failed!')
continue
wordhtml = BeautifulSoup(response.content.decode('gbk', errors='ignore'), "lxml")
explanation = wordhtml.find_all('table')[-3].find_all('tr')
res.append({'word':explanation[0].text.strip(),\
'pinyin': explanation[1].find_all('tr')[0].find_all('td')[1].text.strip(),\
'explanation': explanation[1].find_all('tr')[1].find_all('td')[1].text.strip(),\
'derivation': explanation[1].find_all('tr')[2].find_all('td')[1].text.strip(),\
'example': explanation[1].find_all('tr')[3].find_all('td')[1].text.strip()})
return res
if __name__ == '__main__':
res = downloader('http://www.zd9999.com/cy/')
for i in range(2, 199):
res += downloader(f'http://www.zd9999.com/cy/index_{i}.htm')
print(len(res))
with open('chengyu.json', mode='w+', encoding='utf-8') as json_file:
json.dump(res, json_file, ensure_ascii=False)
scripts/ci.py
+72
View File
@@ -0,0 +1,72 @@
"""
author: pwxcoo
date: 2018-08-02
description: 多线程抓取下载词语并保存
"""
import requests, csv
from bs4 import BeautifulSoup
import time
from multiprocessing.dummy import Pool as ThreadPool
def downloader(url):
"""
下载词语并保存
"""
res = []
try:
response = requests.get(url)
if response.status_code != 200:
print(f'{url} is failed!')
return len(res)
print(f'{url} is parsing')
html = BeautifulSoup(response.content.decode('gbk', errors='ignore'), "lxml")
a = html.find_all('a', target="_blank")
prefix = 'http://www.zd9999.com'
words = [prefix + w.get('href') for w in a]
for i in range(0, len(words)):
print(f'{[words[i]]} is parsing')
try:
response = requests.get(words[i])
wordhtml = BeautifulSoup(response.content.decode('gbk', errors='ignore').replace('<br/>', '\n').replace('<br>', '\n')\
, "lxml")
td = wordhtml.find_all('table')[5].find_all('td')
res.append([td[0].text.strip(), td[1].text.strip()])
except Exception as e:
with open('../data/error.csv', mode='a+', encoding='utf-8', newline='') as error_file:
csv.writer(error_file).writerows([0, e, words[i]])
print(f'{words[i]} is failed! {e}')
continue
except Exception as e:
with open('../data/error.csv', mode='a+', encoding='utf-8', newline='') as error_file:
csv.writer(error_file).writerows([1, e, url])
print(f'{url} is failed! {e}')
with open('../data/ci.csv', mode='a+', encoding='utf-8', newline='') as csv_file:
csv.writer(csv_file).writerows(res)
return len(res)
if __name__ == '__main__':
start_time = time.time()
pool = ThreadPool(100)
urls = ['http://www.zd9999.com/ci/index.htm']
for i in range(2, 1959):
urls.append(f'http://www.zd9999.com/ci/index_{i}.htm')
responses = pool.map(downloader, urls)
pool.close()
pool.join()
end_time = time.time()
print(f'总共耗时 {end_time - start_time}, 抓取了 {sum(responses)} 条数据')
scripts/clean.ipynb
+114
View File
@@ -0,0 +1,114 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Clean Data (Duplicate Removal)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import json"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with open(\"/home/pwxcoo/projects/chinese-xinhua/archived/idiom.json\") as fp:\n",
" idioms = json.load(fp)\n",
"\n",
"len(idioms)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"idioms[:5]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"unique_idioms = set(idiom[\"word\"] for idiom in idioms)\n",
"unique_idioms"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"len(unique_idioms)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"check = dict()\n",
"for idiom in idioms:\n",
" if idiom[\"word\"] in check and str(idiom) > str(check[idiom[\"word\"]]):\n",
"# print(idiom)\n",
"# print(check[idiom[\"word\"]])\n",
" check[idiom[\"word\"]] = idiom\n",
" else:\n",
" check[idiom[\"word\"]] = idiom\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with open(\"/home/pwxcoo/projects/chinese-xinhua/data/idiom-clean.json\", \"w+\", encoding=\"utf-8\") as fp:\n",
" json.dump(list(check.values()), fp, ensure_ascii=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
scripts/word.py
+58
View File
@@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
"""
author: pwxcoo
date: 2018-02-05
description: 抓取下载汉字并保存
"""
import requests,json
from bs4 import BeautifulSoup
def downloader(url):
"""
下载汉字并保存
"""
response = requests.get(url)
if response.status_code != 200:
print(f'{url} is failed!')
return
print(f'{url} is parsing')
html = BeautifulSoup(response.content.decode('gbk', errors='ignore'), "lxml")
a = html.find_all('a', target="_blank")
prefix = 'http://www.zd9999.com'
words = [prefix + w.get('href') for w in a]
res = []
for i in range(0, len(words)):
response = requests.get(words[i])
print(f'{[words[i]]} is parsing')
if response.status_code != 200:
print(f'{words[i]} is failed!')
continue
wordhtml = BeautifulSoup(response.content.decode('gbk', errors='ignore').replace('<br/>', '\n').replace('<br>', '\n')\
, "lxml")
td = wordhtml.find_all('table')[4].find_all('td')
explanation = td[12].text.strip()
res.append({'word': td[1].text.strip(),\
'oldword': td[4].text.strip(),\
'strokes': td[6].text.strip(),\
'pinyin': td[8].text.strip(),\
'radicals': td[10].text.strip(),\
'explanation': explanation[explanation.find('\r\n'):].strip(),\
'more': td[14].text.strip()})
return res
if __name__ == '__main__':
res = downloader('http://www.zd9999.com/zi/index.htm')
for i in range(2, 102):
res += downloader(f'http://www.zd9999.com/zi/index_{i}.htm')
print(len(res))
with open('word.json', mode='w+', encoding='utf-8') as json_file:
json.dump(res, json_file, ensure_ascii=False)
scripts/xiehouyu.py
+36
View File
@@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
"""
author: pwxcoo
date: 2018-02-04
description: 抓取下载歇后语并保存
"""
import requests, json
from bs4 import BeautifulSoup
destination_site = ['http://xhy.5156edu.com/html2/xhy.html', 'http://xhy.5156edu.com/html2/xhy_2.html']
def downloader(url):
"""
下载歇后语并保存
"""
response = requests.get(url)
if response.status_code != 200:
print(f'{url} is failed!')
return
print(f'{url} is parsing')
html = BeautifulSoup(response.content.decode('gbk'), "lxml")
tr = html.find('table', style='word-break:break-all').find_all('tr')
return [{'riddle':t.find_all('td')[0].text, 'answer':t.find_all('td')[1].text} for t in tr][1:]
if __name__ == '__main__':
res = downloader('http://xhy.5156edu.com/html2/xhy.html')
for i in range(2, 282):
res += downloader(f'http://xhy.5156edu.com/html2/xhy_{i}.html')
print(len(res))
with open('xiehouyu.json', mode='w+', encoding='utf-8') as json_file:
json.dump(res, json_file, ensure_ascii=False)