電影天堂爬取詳情頁

爬取電影天堂最新電影,地址https://www.dytt8.net/html/gndy/dyzz/list_23_1.html

import requests
from lxml import etree

BASE_DOMAIN = 'https://www.dytt8.net'
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}


def get_detail_urls(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    html = etree.HTML(text)
    detail_urls = html.xpath('//table[@class="tbspan"]//a/@href')
    detail_urls = map(lambda url: BASE_DOMAIN+url, detail_urls)
    return detail_urls


def get_detail_info(url):
    movie = {}
    response = requests.get(url, headers=HEADERS)
    text = response.content.decode(encoding='gbk', errors='ignore')
    html = etree.HTML(text)
    movie['title'] = html.xpath('//div[@class="title_all"]//font/text()')[0]
    img = html.xpath("//div[@id='Zoom']//img/@src")
    movie['cover'] = img[0]
    movie['screenshot'] = img[1]
    infos = html.xpath('//div[@id="Zoom"]//p/text()')
    # 提取信息
    is_actors = False
    actors = []
    for info in infos:
        # print(info)
        if info.startswith('◎年　　代'):
            movie['year'] = info.replace("◎年　　代", "").strip()
        elif info.startswith('◎產　　地'):
            movie['country'] = info.replace("◎產　　地", "").strip()
        elif info.startswith('◎類　　別'):
            movie['category'] = info.replace("◎類　　別", "").strip()
        elif info.startswith('◎豆瓣評分'):
            movie['douban_rating'] = info.replace("◎豆瓣評分", "").strip()
        elif info.startswith('◎片　　長'):
            movie['duration'] = info.replace("◎片　　長", "").strip()
        elif info.startswith('◎導　　演'):
            movie['director'] = info.replace("◎導　　演", "").strip()
        elif info.startswith('◎主　　演'):
            actors = [info.replace("◎主　　演", "").strip()]
            is_actors = True
        elif is_actors:
            if info.startswith('◎'):
                is_actors = False
                movie['actors'] = actors
                continue
            actors.append(info.strip())
    movie['download'] = html.xpath("//div[@id='Zoom']//tbody//a/text()")
    movie['magnet'] = html.xpath("//div[@id='Zoom']//a/@href")[0]

    return movie


def spider():
    base_url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html'
    movies = []
    for i in range(1, 2):
        url = base_url.format(i)
        detail_urls = get_detail_urls(url)
        for detail_url in detail_urls:
            # 對詳情頁提取信息
            # print(detail_url)
            movies.append(get_detail_info(detail_url))
    print(movies)


if __name__ == '__main__':
    spider()

學習的視頻中代碼有幾處跟我的有不同,可以學習

一、

其中提取主演的代碼不同,如下

for index,info in enumerate(infos):
        if info.startswith("◎年　　代"):
            info = parse_info(info,"◎年　　代")
            movie['year'] = info
        # .......省略
        elif info.startswith("◎主　　演"):
            info = parse_info(info,"◎主　　演")
            actors = [info]
            for x in range(index+1,len(infos)):
                actor = infos[x].strip()
                if actor.startswith("◎"):
                    break
                actors.append(actor)
            movie['actors'] = actors

采用的是index的方式.

enumerate() 函數用於將一個可遍歷的數據對象(如列表、元組或字符串)組合為一個索引序列，同時列出數據和數據下標，一般用在 for 循環當中。

 

二、

還有

detail_urls = map(lambda url:BASE_DOMAIN+url,detail_urls)

這段代碼沒怎么用過.記錄下.

三、

base_url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html'
    for i in range(1, 10):
        url = base_url.format(i)

以前寫的時候沒這么寫過，都是直接弄成

url = 'https://www.dytt8.net/html/gndy/dyzz/list_23_'+i+'.html'這樣.

四、

在自己寫代碼時有個錯誤,也需要記錄下,在movie的字典賦值的時候,

如下

    for info in infos:
        # print(info)
        if info.startswith('◎年　　代'):
            year = info.replace("◎年　　代", "").strip()
        elif info.startswith('◎產　　地'):
            country = info.replace("◎產　　地", "").strip()
        elif info.startswith('◎類　　別'):
            category = info.replace("◎類　　別", "").strip()
        elif info.startswith('◎豆瓣評分'):
            douban_rating = info.replace("◎豆瓣評分", "").strip()

賦值的時候使用

 movie = {
        'year': year,
        'country': country,
        'category': category,
        'douban_rating': douban_rating
    }

會報錯,因為其中有一個豆瓣評分是不存在的,不會對其賦值,所以movie賦值的時候會錯誤.

UnboundLocalError: local variable 'douban_rating' referenced before assignment

 當然，個人覺得用正則可以更容易解決。

在記錄下爬取豆瓣正在上映的電影用xpath的代碼

import requests
from lxml import etree

headers = {
    'Referer': 'https://movie.douban.com/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
url = "https://movie.douban.com/cinema/nowplaying/changsha/"
response = requests.get(url, headers=headers)
text = response.text
html = etree.HTML(text)
lis = html.xpath("//div[@id='nowplaying']//ul[@class='lists']/li")
movies = []
for li in lis:
    # print(etree.tostring(li, encoding='utf-8').decode('utf-8'))
    title = li.xpath("@data-title")[0]
    score = li.xpath("@data-score")[0]
    duration = li.xpath("@data-duration")[0]
    director = li.xpath("@data-director")[0]
    actor = li.xpath("@data-actors")[0]
    img = li.xpath(".//img/@src")[0]
    movie = {
        'title': title,
        'score': score,
        'duration': duration,
        'director': director,
        'actor': actor,
        'img': img
    }
    movies.append(movie)

print(movies)