python-requests爬取數據保存至數據庫

import requests
from fake_useragent import UserAgent
from lxml import etree
import pymysql


conn = pymysql.connect(host='47.107.190.1', port=3306, user='zhangsan',
                       password='Changeme_123', database='qiubai', charset='utf8')
# 創建游標
cursor = conn.cursor()

url = 'https://www.qiushibaike.com/text/'

ua = UserAgent()

headers = {
        'User-Agent': ua.random
}

response = requests.get(url=url, headers=headers)
# print(response.text)

# 解析數據
html = etree.HTML(response.text)
# 先拿到存放段子的div
divs = html.xpath('//div[contains(@id, "qiushi_tag_")]')
# print(len(divs))
# 循環從每一個div中提取需要的信息
for div in divs:
    # 定位到具體元素之后, 在當前元素下提取內容,一定要寫./
    img = div.xpath('.//div[contains(@class, "author")]/a[1]/img/@src')[0]
    author = div.xpath('string(.//div[contains(@class, "author")]/a[2])').strip()
    detail_url = 'https://www.qiushibaike.com' + div.xpath('.//a[@class="contentHerf"]/@href')[0]
    # print(detail_url)
    # 進入詳情頁,爬取詳情頁內容
    detail_response = requests.get(url=detail_url, headers=headers)
    detail_html = etree.HTML(detail_response.text)
    content = detail_html.xpath('string(//div[@class="content"])').strip()
    # print(content)

    # 把爬下來的內容存入到數據庫.
    try:
        sql = 'insert into duanzi(img, author, detail_url, content) values("%s", "%s", "%s", "%s")'
        cursor.execute(sql, (img, author, detail_url, content))
        # 提交
        conn.commit()
    except Exception as e:
        print(e)
        conn.rollback()

cursor.close()
conn.close()