python爬愛奇藝電視劇及劇集鏈接

 爬取的顯示結果如下：

 

話不多說，下面是python代碼。。。。。。。。。。。。。。。

import requests
import re
from bs4 import BeautifulSoup
import json

if __name__ == '__main__':

    for i in range(1,10):#翻頁數可自行選擇
        #獲取URL，並自動翻頁
        url = 'http://list.iqiyi.com/www/2/-------------11-'+str(i)+'-1-iqiyi--.html'

        headers = {
            'Access-Control-Allow-Credentials': 'true',
            'Cache-Control': 'max-age=900',
            'Content-Encoding': 'gzip',
            'Content-Language': 'zh-CN',
            'Content-Type': 'text/html; charset=UTF-8',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
            'Referer': 'http://list.iqiyi.com/www/2/-------------11-4-1-iqiyi--.html',
            'Upgrade-Insecure-Requests': '1'
        }
        target = requests.get(url=url,headers=headers).text
        #爬取愛奇藝電視劇總網頁，解析HTML網頁
        soup = BeautifulSoup(target,'html.parser') #html.parser解析HTML網頁
        returnSoup = soup.find_all("div", attrs={"class": "wrapper-piclist"})[0]
        returnSoup1= str(returnSoup).replace('\r\n','').replace('\n','').replace(' ','').replace('"rseat="bigTitle','')
        href_title = re.findall('"data-widget-qidanadd="qidanadd"href="(.*?)"target="_blank"title=".*?"><imgalt="(.*?)"height="236"rseat="dsjp7"src=',str(returnSoup1))


        for i in href_title:
            href=i[0]
            title=i[1]

            #轉到某個電視劇鏈接網頁，並解析
            href1=str(href).split('#')[0]
            url2=href1
            target2 = requests.get(url=url2).text
            soup2 = BeautifulSoup(target2,'html.parser')
            returnsoup2 = soup2.find_all('div',attrs={'class':'site-piclist_pic'})

            #用正則表達式獲取劇集鏈接
            result2 = re.findall('(?<=href=\").*?(?=\")',str(returnsoup2))
            #用正則表達式獲取劇集名稱
            title2 = re.findall('(?<=title=\").*?(?=\">)',str(returnsoup2))
            j=len(title2)
            #輸出爬取結果
            for i in range(1,j-2):
                str1='第'+str(i)+'集'
                print(result2[i])
                print(str1,title2[i])