初識python 之 爬蟲：爬取豆瓣電影最熱評論

主要用到lxml的etree解析網頁代碼，xpath獲取HTML標簽。

代碼如下：

#!/user/bin env python
# author:Simple-Sir
# time:2019/7/17 22:08
# 獲取豆瓣網正在上映電影最熱評論
import requests
from lxml import etree

# 偽裝瀏覽器
headers ={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    'Referer':'https://movie.douban.com/'
}
# 獲取首頁網頁信息並解析
url = 'https://movie.douban.com/cinema/nowplaying/chengdu/'

def getUrlText(url):
    respons = requests.get(url,headers=headers)  # 獲取網頁信息
    urlText = respons.text
    html = etree.HTML(urlText)  # 使用lxml解析網頁
    return html

# 提取電影名稱及詳情地址鏈接列表
def getWallUrl(url):
    hrefUrl = getUrlText(url)
    ul = hrefUrl.xpath('//ul[@class="lists"]')[0]  # 獲取ul標簽
    liList = ul.xpath('./li')  # # 獲取li標簽列表
    liHrefs = []
    for li in liList:
        liHref = li.xpath('.//@href')[0]
        name = li.xpath('@data-title')[0]
        msg = {
            name:liHref
        }
        liHrefs.append(msg)
    return liHrefs

# 解析電影詳情地址
def downPL(url):
    moveUrl = getWallUrl(url)
    n=0
    for murl in moveUrl:
        n+=1
        for d in murl:
            plHtml = getUrlText(murl[d])
            plTextFull = plHtml.xpath('//div[@id="hot-comments"]//span[@class="hide-item full"]//text()')
            plTextShort = plHtml.xpath('//div[@id="hot-comments"]//span[@class="short"]//text()')
            if(len(plTextFull)==0 and len(plTextShort)>0):
                print('正在寫入《{}》的評論。'.format(d))
                with open('豆瓣評論.txt','a+',encoding='utf-8') as fp:
                    fp.write('{}、《{}》的最熱評論是：\n{}\n\n'.format(n,d,plTextShort[0]))
            elif(len(plTextFull)>0):
                print('正在寫入《{}》的評論。'.format(d))
                with open('豆瓣評論.txt','a+',encoding='utf-8') as fp:
                    fp.write('{}、《{}》的最熱評論是：\n{}\n\n'.format(n,d,plTextShort[0]))
            else:
                print('正在寫入《{}》的評論。'.format(d))
                with open('豆瓣評論.txt','a+',encoding='utf-8') as fp:
                    fp.write('{}、《{}》暫無評論！\n\n'.format(n,d))
    return print('{}部電影的所有評論已全部寫入“豆瓣評論.txt”，請查看。'.format(n))

if __name__ == '__main__':
    downPL(url)

 

執行效果：

 

 文件詳情：