mirror: 初始快照

scripts/ci.py

@@ -0,0 +1,72 @@
+"""
+
+author: pwxcoo
+date: 2018-08-02 
+description: 多线程抓取下载词语并保存
+
+"""
+
+import requests, csv
+from bs4 import BeautifulSoup
+import time
+from multiprocessing.dummy import Pool as ThreadPool
+
+def downloader(url):
+    """
+    下载词语并保存
+    """
+    res = []
+    try:
+        response = requests.get(url)
+        
+        if response.status_code != 200:
+            print(f'{url} is failed!')
+            return len(res)
+
+        print(f'{url} is parsing')
+        html = BeautifulSoup(response.content.decode('gbk', errors='ignore'), "lxml")
+        a = html.find_all('a', target="_blank")
+        
+        prefix = 'http://www.zd9999.com'
+        words = [prefix + w.get('href') for w in a]
+    
+        for i in range(0, len(words)):
+            print(f'{[words[i]]} is parsing')
+
+            try:
+                response  = requests.get(words[i])
+                wordhtml = BeautifulSoup(response.content.decode('gbk', errors='ignore').replace('<br/>', '\n').replace('<br>', '\n')\
+                            , "lxml")
+                td = wordhtml.find_all('table')[5].find_all('td')
+                res.append([td[0].text.strip(), td[1].text.strip()])
+     
+            except Exception as e:
+                with open('../data/error.csv', mode='a+', encoding='utf-8', newline='') as error_file:
+                    csv.writer(error_file).writerows([0, e, words[i]])
+                print(f'{words[i]} is failed! {e}')
+                continue
+
+    except Exception as e:
+        with open('../data/error.csv', mode='a+', encoding='utf-8', newline='') as error_file:
+            csv.writer(error_file).writerows([1, e, url])
+        print(f'{url} is failed! {e}')
+
+    with open('../data/ci.csv', mode='a+', encoding='utf-8', newline='') as csv_file:
+        csv.writer(csv_file).writerows(res)
+    
+    return len(res)
+
+if __name__ == '__main__':
+    start_time = time.time()
+
+    pool = ThreadPool(100)
+    urls = ['http://www.zd9999.com/ci/index.htm']
+    for i in range(2, 1959):
+        urls.append(f'http://www.zd9999.com/ci/index_{i}.htm')
+    responses = pool.map(downloader, urls)
+    pool.close()
+    pool.join()
+     
+    end_time = time.time()
+    
+    print(f'总共耗时 {end_time - start_time}, 抓取了 {sum(responses)} 条数据')