python爬蟲爬取豆瓣電視劇數據

作為一個python小白，在下面的問題中出錯：

1.因為豆瓣頁面的數據加載涉及到異步加載，所以需要通過瀏覽器獲取到真正的網頁鏈接。

2.將字典轉化為DataFrame以后寫入.csv文件。DataFrame是一個表單一樣的數據結構。

3.從網頁獲取的json數據的處理。

代碼：

import re
import requests
from bs4 import BeautifulSoup
import time
import random
import string
import logging
import json
import jsonpath
import pandas as pd

import pdb

User_Agents = [
    'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
    'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
    'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
    'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
    'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
]


class DoubanSpider(object):
    '''豆瓣爬蟲'''

    def __init__(self):
        # 基本的URL
        self.base_url = 'https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_' + 'start={start}'
        self.full_url = self.base_url
        self.tv_detailurl = 'https://movie.douban.com/j/subject_abstract?subject_id='

    def download_tvs(self, offset):
        # offset控制一次下載的量，resp返回的響應體
        self.headers = {'User-Agent': random.choice(User_Agents)}
        self.full_url = self.base_url.format(start=offset)
        resp = None
        try:
            resp = requests.get(self.full_url, headers=self.headers)
        except Exception as e:
            print(logging.error(e))
        return resp

    def get_tvs(self, resp):
        # resp響應體
        # movies爬取到的電影信息
        print('get_tvs')
        print(resp)
        tv_urls = []
        if resp:
            if resp.status_code == 200:
                html = resp.text
                unicodestr = json.loads(html)
                tv_list = unicodestr['subjects']
                for item in tv_list:
                    data = re.findall(r'[0-9]+', str(item['url']))
                    tv_urls.append(self.tv_detailurl + str(data[0]))
            print('tv_urls')
            return tv_urls
        return None

    def download_detailtvs(self, tv_urls):
        tvs = []
        for item in tv_urls:
            self.headers = {'User-Agent': random.choice(User_Agents)}
            resp = requests.get(item, headers=self.headers)
            html = resp.text
            unicodestr = json.loads(html)
            tvs.append(unicodestr['subject'])
        return tvs


def main():
    spider = DoubanSpider()
    offset =0
    data = {'title':[], 'types':[], 'directors':[], 'actors':[], 'release_year':[], 'region':[], 'star':[], 'episodes_count':[],'rate':[]}
    tv_file = pd.DataFrame(data)
    tv_file.to_csv('res_url.csv',mode='w',index=None)
    while True:
        reps = spider.download_tvs(offset)
        tv_urls = spider.get_tvs(reps)
        tvss = spider.download_detailtvs(tv_urls)
        for tvsss in tvss:
            '''
            #pdb.set_trace()
            tvsss=re.sub(r'\\u200e','',tvsss)
            tvsss = re.sub(r'\'', '\"', tvsss)
            tvsss = re.sub(r'\'', '\"', tvsss)
            #將short_comment去掉
            tvsss = re.sub(r'(\"short_comment\").*(\"directors\")', '\"directors\"',tvsss)
            #將true,false改為"True","False"
            tvsss = re.sub(r'True', '\"True\"', tvsss)
            tvsss = re.sub(r'False', '\"False\"', tvsss)
            #給所有的list加上雙引號
            print(tvsss)
            #將: [轉化為: "[
            tvsss = re.sub(r': \[', ': "[', tvsss)
            #jiang ],zhuanhuawei ]",
            tvsss=re.sub(r'\],',']",',tvsss)
            # 以上正確
            print(tvsss)
            #將director的內容改為單引號
            r1 = re.findall(r'(?<=directors": "\[).*?(?=\]\")', tvsss)
            #正確
            if r1:
                r2 = re.sub(r'\"', '\'', r1[0])
                r3 = re.sub(r'\"', '\'', r2)
                tvsss = re.sub(r'(?<=directors\": \"\[).*?(?=\]\")', r3, tvsss)
            #zhengque
            #將actors的內容改為單引號
            print(tvsss)
            r1 = re.findall(r'(?<=actors\": \"\[).*?(?=\]\")', tvsss)
            print("actors")
            print(r1)
            if r1:
                r2 = re.sub(r'\"', '\'', r1[0])
                r3 = re.sub(r'\"', '\'', r2)
                tvsss = re.sub(r'(?<=actors\": \"\[)[\s\S]*?(?=\]\")', r3, tvsss)
            #將劇情types改為單引號
            print(tvsss)
            r1 = re.findall(r'(?<=types": "\[).*?(?=\]\")', tvsss)
            if r1:
                r2 = re.sub(r'\"', '\'', r1[0])
                r3 = re.sub(r'\"', '\'', r2)
                tvsss = re.sub(r'(?<=types\": \"\[).*?(?=\]\")', r3, tvsss)
            # 正確
            #將二維的數據轉化為一維的
            types=str(tvs['types'])
            actor = str(tvs['actors'])
            director = str(tvs['directors'])
            types=re.sub(r'\'','',types)
            actor = re.sub(r'\'', '', actor)
            director = re.sub(r'\'', '', director)
            types = re.sub(r'\'', '', types)
            actor= re.sub(r'\'', '', actor)
            director = re.sub(r'\'', '', director)
            types=types.strip('[]')
            actor=actor.strip('[]')
            director=director.strip('[]')

            data2={'title':tvs['title'], 'types':types, 'directors':director, 'actors':actor, 'release_year':tvs['release_year'], 'region':tvs['region'], 'star':tvs['star'], 'episodes_count':tvs['episodes_count'],'rate':tvs['rate']}
            print(data2)
            tv_file=pd.DataFrame(data2,index=[0])
            #tv_file = pd.DataFrame(data)
            # pdb.set_trace()
            tv_file.to_csv('res_url.csv', mode='a', index=None,header=None)
        offset += 20
        id = offset
        # 控制訪問速速
        time.sleep(10)


if __name__ == '__main__':
    main()