python爬蟲---抓取優酷的電影

最近在學習爬蟲，用的BeautifulSoup4這個庫，設想是把優酷上面的電影的名字及鏈接爬到，然后存到一個文本文檔中。比較簡單的需求，第一次寫爬蟲。貼上代碼供參考：

# coding:utf-8

import requests
import os
from bs4 import BeautifulSoup
import re
import time

'''抓優酷網站的電影：http://www.youku.com/ '''

url = "http://list.youku.com/category/show/c_96_s_1_d_1_u_1.html"
h = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0"}




#存到movie文件夾的文本文件中
def write_movie():
    currentPath = os.path.dirname(os.path.realpath(__file__))
    #print(currentPath)
    moviePath = currentPath + "\\" + "movie"+"\\" + "youku_movie_address.text"
    #print(moviePath)
    fp = open(moviePath ,encoding="utf-8",mode="a")

    for x in list_a:
        text = x.get_text()
        if text == "":
            try:
                fp.write(x["title"] + ":    " + x["href"]+"\n")
            except IOError as msg:
                print(msg)

    fp.write("-------------------------------over-----------------------------" + "\n")
    fp.close()

#第一頁
res = requests.get(url,headers = h)
print(res.url)
soup = BeautifulSoup(res.content,'html.parser')
list_a = soup.find_all(href = re.compile("==.html"),target="_blank")
write_movie()

for num in range(2,1000):

    #獲取“下一頁”的href屬性
    fanye_a = soup.find(charset="-4-1-999" )
    fanye_href = fanye_a["href"]
    print(fanye_href)
    #請求頁面
    ee = requests.get("http:" + fanye_href,headers = h)
    time.sleep(3)
    print(ee.url)

    soup = BeautifulSoup(ee.content,'html.parser')
    list_a = soup.find_all(href = re.compile("==.html"),target="_blank")

    #調用寫入的方法
    write_movie()
    time.sleep(6)

運行后的txt內的文本內容：