一個小說的爬蟲，帶GUI界面的
主要功能
1.多線程提取 可使用代{過}{}過濾理
2. 實時輸出過程
3. 一本書的txt文件
使用方法
很多人學習蟒蛇，不知道從何學起。 很多人學習尋找python，掌握了基本語法之后，不知道在哪里案例上手。 很多已經可能知道案例的人，卻不怎么去學習更多高深的知識。 這三類人，我給大家提供一個好的學習平台，免費獲取視頻教程，電子書，以及課程的源代碼！ QQ群：101677771 歡迎加入，一起討論學習
1. 首先配置好python3環境，
2.新建一個空目錄，在此目錄下要新建start.py文件，將源碼復制在start.py文件內
3. 在此目錄下打開 cmd.exe，執行命令 python -m venv 。
4.分別
pip install requests pip install pip install pip install pyinstaller
5.在
pyinstaller - F -w - *.py
結束后在 dist 看到下 exe已執行打包好
exe 執行文件下載
鏈接：
https://pan
.baidu.com/s/10FcAcJ.mv8Blx3evX4TQ?pwdh 提取碼j63v6h=3vdh
import time import requests import os import re import random from lxml import etree import webbrowser import PySimpleGUI as sg import threading # user-agent header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36" } # 代{過}{濾}理 proxies = {} # 刪除書名中特殊符號 # 筆趣閣基地址 baseurl = 'https://www.xbiquwx.la/' # 線程數量 threadNum = 6 pool_sema = None THREAD_EVENT = '-THREAD-' cjstatus = False # txt存儲目錄 filePath = os.path.abspath(os.path.join(os.getcwd(), 'txt')) if not os.path.exists(filePath): os.mkdir(filePath) # 刪除特殊字符 def deletetag(text): return re.sub(r'[\[\]#\/\\:*\,;\?\"\'<>\|\(\)《》&\^!~=%\{\}@！：。·！￥……（） ]','',text) # 入口 def main(): global cjstatus, proxies, threadNum, pool_sema sg.theme("reddit") layout = [ [sg.Text('輸入要爬取的小說網址，點此打開筆趣閣站點復制', font=("微軟雅黑", 12), key="openwebsite", enable_events=True, tooltip="點擊在瀏覽器中打開")], [sg.Text("小說目錄頁url,一行一個:")], [ sg.Multiline('', key="url", size=(120, 6), autoscroll=True, expand_x=True, right_click_menu=['&Right', ['粘貼']] ) ], [sg.Text(visible=False, text_color="#ff0000", key="error")], [ sg.Button(button_text='開始采集', key="start", size=(20, 1)), sg.Button(button_text='打開下載目錄', key="opendir", size=(20, 1), button_color="#999999") ], [sg.Text('填寫ip代{過}{濾}理，有密碼格式 用戶名:密碼@ip:端口，無密碼格式 ip:端口。如 demo:123456@123.1.2.8:8580')], [ sg.Input('', key="proxy"), sg.Text('線程數量:'), sg.Input('5', key="threadnum"), ], [ sg.Multiline('等待采集', key="res", disabled=True, border_width=0, background_color="#ffffff", size=( 120, 6), no_scrollbar=False, autoscroll=True, expand_x=True, expand_y=True, font=("宋體", 10), text_color="#999999") ], ] window = sg.Window('采集筆趣閣小說', layout, size=(800, 500), resizable=True,) while True: event, values = window.read() if event == sg.WIN_CLOSED or event == 'close': # if user closes window or clicks cancel break if event == "openwebsite": webbrowser.open('%s' % baseurl) elif event == 'opendir': os.system('start explorer ' + filePath) elif event == 'start': if cjstatus: cjstatus = False window['start'].update('已停止...點擊重新開始') continue window['error'].update("", visible=False) urls = values['url'].strip().split("\n") lenth = len(urls) for k, url in enumerate(urls): if (not re.match(r'%s\d+_\d+/' % baseurl, url.strip())): if len(url.strip()) > 0: window['error'].update("地址錯誤:%s" % url, visible=True) del urls[k] if len(urls) < 1: window['error'].update( "每行地址需符合 %s84_84370/ 形式" % baseurlr, visible=True) continue # 代{過}{濾}理 if len(values['proxy']) > 8: proxies = { "http": "http://%s" % values['proxy'], "https": "http://%s" % values['proxy'] } # 線程數量 if values['threadnum'] and int(values['threadnum']) > 0: threadNum = int(values['threadnum']) pool_sema = threading.BoundedSemaphore(threadNum) cjstatus = True window['start'].update('采集中...點擊停止') window['res'].update('開始采集') for url in urls: threading.Thread(target=downloadbybook, args=( url.strip(), window,), daemon=True).start() elif event == "粘貼": window['url'].update(sg.clipboard_get()) print("event", event) if event == THREAD_EVENT: strtext = values[THREAD_EVENT][1] window['res'].update(window['res'].get()+"\n"+strtext) cjstatus = False window.close() #下載 def downloadbybook(page_url, window): try: bookpage = requests.get(url=page_url, headers=header, proxies=proxies) except Exception as e: window.write_event_value( '-THREAD-', (threading.current_thread().name, '\n請求 %s 錯誤，原因:%s' % (page_url, e))) return if not cjstatus: return # 鎖線程 pool_sema.acquire() if bookpage.status_code != 200: window.write_event_value( '-THREAD-', (threading.current_thread().name, '\n請求%s錯誤，原因:%s' % (page_url, page.reason))) return bookpage.encoding = 'utf-8' page_tree = etree.HTML(bookpage.text) bookname = page_tree.xpath('//div[@id="info"]/h1/text()')[0] bookfilename = filePath + '/' + deletetag(bookname)+'.txt' zj_list = page_tree.xpath( '//div[@class="box_con"]/div[@id="list"]/dl/dd') for _ in zj_list: if not cjstatus: break zjurl = page_url + _.xpath('./a/@href')[0] zjname = _.xpath('./a/@title')[0] try: zjpage = requests.get( zjurl, headers=header, proxies=proxies) except Exception as e: window.write_event_value('-THREAD-', (threading.current_thread( ).name, '\n請求%s:%s錯誤，原因:%s' % (zjname, zjurl, zjpage.reason))) continue if zjpage.status_code != 200: window.write_event_value('-THREAD-', (threading.current_thread( ).name, '\n請求%s:%s錯誤，原因:%s' % (zjname, zjurl, zjpage.reason))) return zjpage.encoding = 'utf-8' zjpage_content = etree.HTML(zjpage.text).xpath('//div[@id="content"]/text()') content = "\n【"+zjname+"】\n" for _ in zjpage_content: content += _.strip() + '\n' with open(bookfilename, 'a+', encoding='utf-8') as fs: fs.write(content) window.write_event_value( '-THREAD-', (threading.current_thread().name, '\n%s:%s 采集成功' % (bookname, zjname))) time.sleep(random.uniform(0.05, 0.2)) # 下載完畢 window.write_event_value('-THREAD-', (threading.current_thread( ).name, '\n請求 %s 結束' % page_url)) pool_sema.release() if __name__ == '__main__': main()