python，爬取小說網站小說內容，同時每一章存在不同的txt文件中

 

 

思路，第一步小說介紹頁獲取章節地址，第二部訪問具體章節，獲取章節內容

具體如下：先獲取下圖章節地址

 

 

def stepa(value,headers):
    lit=[]
    response = requests.get(value, headers=headers)
    html = etree.HTML(response.text)
    url = html.xpath('//*[@id="chapterlist"]//@href')#獲取每章地址
    lit.append(url)
    return(lit)
add=stepa(value,headers)
allurl=add[0]#去掉括號

上方代碼可獲取到下圖紅色區域內內容，即每一章節地址的變量部分，且全部存在腳本輸出的集合中

 

 第二部，循環訪問集合中的章節地址，獲取章節內容，同時逐行存儲在對應章節命名的txt文件中

for link in allurl:
    link = 'http://www.666wx.cc'+link#拼接地址，可訪問
    response = requests.get(link, headers=headers)
    html = etree.HTML(response.text)
    name = html.xpath('//*[@id="center"]/div[1]/h1/text()')#章節
    name =name[0]
    content = html.xpath('//*[@id="content"]/text()')#章節內容
    for 內容 in content:
        內容 = 內容.strip()#去掉每行后的換行符
        with open(path+'\\'+str(name)+'.txt', 'a',encoding='utf-8') as w:
            w.write(str(內容))
            w.close()

生成的文件一覽

 

 txt內容

 

 全部腳本

# -*-coding:utf8-*-
# encoding:utf-8
#本腳本爬取http://www.666wx.cc站小說
import requests
from lxml import etree
import os
import sys
import re

headers = {
    'authority': 'cl.bc53.xyz',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'sec-fetch-site': 'none',
    'sec-fetch-mode': 'navigate',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cookie': '__cfduid=d9b8dda581516351a1d9d388362ac222c1603542964',
}    

value = "http://www.666wx.cc/txt/77079/"#小說地址

path = os.path.abspath(os.path.dirname(sys.argv[0]))

def stepa(value,headers):
    lit=[]
    response = requests.get(value, headers=headers)
    html = etree.HTML(response.text)
    url = html.xpath('//*[@id="chapterlist"]//@href')#獲取每章地址
    lit.append(url)
    return(lit)
add=stepa(value,headers)
allurl=add[0]#去掉括號
for link in allurl:
    link = 'http://www.666wx.cc'+link#拼接地址，可訪問
    response = requests.get(link, headers=headers)
    html = etree.HTML(response.text)
    name = html.xpath('//*[@id="center"]/div[1]/h1/text()')#章節
    name =name[0]
    content = html.xpath('//*[@id="content"]/text()')#章節內容
    for 內容 in content:
        內容 = 內容.strip()#去掉每行后的換行符
        with open(path+'\\'+str(name)+'.txt', 'a',encoding='utf-8') as w:
            w.write(str(內容))
            w.close()
print("ok")