python爬取网站信息

 一.爬取网站数据

大体思路，采用requests模块爬取页面源代码，处理网页反爬机制（加入headers模拟人工访问浏览器），再采用re模块进行信信息处理分割，取得我所需要的信息。整合为列表方便下一步处理。

import re
import requests
# 请求页面
url = 'https://b.faloo.com/y_0_1.html'
headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
    'Accept-Encoding': 'gzip, deflate,br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Cookie': 'host4chongzhi=http%3a%2f%2fb.faloo.com%2f; Hm_lvt_6d308f6626f6d0864b6bb4f348f2b5e5=1648727329; curr_url=https%3A//b.faloo.com/y_0_1.html; Hm_lpvt_6d308f6626f6d0864b6bb4f348f2b5e5=1648727414',
    'Host': 'b.faloo.com',
    'Referer': 'https://b.faloo.com/Rank_1.html',
    'Upgrade-Insecure-Requests': 1,
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'
  }
response = requests.get(url).text
# 过滤数据
print(("开始爬取网站数据！"))
a_pattern = re.compile(r'<h1 class="fontSize17andHei" title="(.*?)">')
title = re.findall(a_pattern, response)
# print(title)
print(("小说标题爬取完成！"))
author_pattern = re.compile(r'<span class="fontSize14andsHui">(.*?)</span>')
span_text = re.findall(author_pattern, response)
#print(span_text)

author_list = []
for item in span_text:
    a_title_pattern = re.compile(r'<a.*?>(.*?)</a>')
    a_title = re.findall(a_title_pattern, item)
    #print(a_title)
    image_pattern = re.compile(r'<img.*?')
    if image_pattern.match(a_title[0]) is not None:
        author = re.findall(r'<a.*title="(.*?)" target="_blank">', a_title[0])[0]
    else:
        author = a_title[0]
    author_list.append(author)
# print(len(author_list))
print("作者信息爬取完成！")
novel_type_pattern = re.compile(r'<span class="fontSize14andHui">(.*?)</span>')
novel_type_list = re.findall(novel_type_pattern, response)
novel_type_li = []
for novel_type_name in novel_type_list:
    novel_type = re.findall(r'<a.*title="(.*?)".*', novel_type_name)
    novel_type_li.append(novel_type[0])
# print(len(novel_type_li))
print("文章类型信息爬取完成！")
novel_date = re.compile(r'<font color=".*?">(.*?)</font>')
novel_date_list = re.findall(novel_date, response)
# print(novel_date_list)
print("文章更新爬取完成！")
novel_click = re.compile(r'<span>周点击：(.*?)</span>')
novel_click_list = re.findall(novel_click, response)
# print(novel_click_list)
print("文章周点击数爬取完成！")
novel_number = re.compile(r'<span>字数：(.*?)</span>')
novel_number_list = re.findall(novel_number, response)
# print(novel_number_list)
print("文章字数爬取完成！")
lis=[]
for i in range(len(novel_date_list)):
    li = []
    li.append(title[i])
    li.append(author_list[i])
    li.append(novel_type_li[i])
    li.append(novel_click_list[i])
    li.append(novel_number_list[i])
    li.append(novel_date_list[i])
    lis.append(li)
# print(lis)

 二.将爬取数据存入Execl表格

import xlwt

print("开始创建Execl表格！")
f=xlwt.Workbook()
sheet1 = f.add_sheet('小说',cell_overwrite_ok=True)
row0 = ["书名","作者","小说类型","日期","字数","周点击数"]
for i in range(len(row0)):
    sheet1.write(0,i,row0[i])
for i2 in lis:
    for i3 in range(len(i2)):
        sheet1.write(lis.index(i2)+1,i3,i2[i3])
    f.save('飞卢小说信息网.xls')
print("Execl表格创建完成！")

 三.将数据写入数据库中

import pymysql

# li = [[4,'赵六','物理',97],[5,'孙七','化学',91],[6,'王八','生物',93]]
# create table novel(id int(8) not null auto_increment primary key,
# book_name varchar(100),author varchar(25),book_type varchar(8),
# week_clicks varchar(25),word_counts varchar(30),book_date varchar(10));
con = pymysql.connect(host='192.168.135.165',user='root',password='123456',database='xiaoshuo')
cur = con.cursor()
print("数据库连接成功！")
# cur.execute('use xiaoshu')
for i in lis:
    for j in range(0,1):
        cur.execute("insert into novel values(NULL,'%s','%s','%s','%s','%s','%s')"%(i[0],i[1],i[2],i[3],i[4],i[5]))
cur.execute('select * from novel')
# a=cur.fetchall()
# print(a)
print("数据写入完成！")
con.commit()
con.close()

四.结果

将以上代码整合，就是如下效果