python爬蟲——抓取電影天堂電影信息

做個小練習，抓取的是電影天堂里面最新電影的頁面。鏈接地址：http://www.dytt8.net/html/gndy/dyzz/index.html

首先我們需要獲取里面電影詳情的網頁地址：

import urllib2
import os
import re
import string


# 電影URL集合
movieUrls = []


# 獲取電影列表
def queryMovieList():

    url = 'http://www.dytt8.net/html/gndy/dyzz/index.html'   
    conent = urllib2.urlopen(url)
    conent =  conent.read()
    conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  
    pattern = re.compile ('<div class="title_all"><h1><font color=#008800>.*?</a>></font></h1></div>'+
                          '(.*?)<td height="25" align="center" bgcolor="#F4FAE2"> ',re.S)
    items = re.findall(pattern,conent) 
    
    str = ''.join(items)
    pattern = re.compile ('<a href="(.*?)" class="ulink">(.*?)</a>.*?<td colspan.*?>(.*?)</td>',re.S)
    news = re.findall(pattern, str)

    for  j in news:
       
            movieUrls.append('http://www.dytt8.net'+j[0])

抓取詳情頁中的電影數據

def queryMovieInfo(movieUrls):

    for index, item in enumerate(movieUrls):

        print('電影URL: ' + item)

        conent = urllib2.urlopen(item)
        conent = conent.read()
        conent = conent.decode('gb2312','ignore').encode('utf-8','ignore')  


        movieName = re.findall(r'<div class="title_all"><h1><font color=#07519a>(.*?)</font></h1></div>', conent, re.S)
        if (len(movieName) > 0):
            movieName = movieName[0] + ""
            # 截取名稱
            movieName = movieName[movieName.find("《") + 3:movieName.find("》")]
        else:
            movieName = ""

        print("電影名稱: " + movieName.strip())

        movieContent = re.findall(r'<div class="co_content8">(.*?)</tbody>',conent , re.S)


        pattern = re.compile('<ul>(.*?)<tr>', re.S)
        movieDate = re.findall(pattern,movieContent[0])

        if (len(movieDate) > 0):
            movieDate = movieDate[0].strip() + ''
        else:
            movieDate = ""

        print("電影發布時間: " + movieDate[-10:])

        pattern = re.compile('<br /><br />(.*?)<br /><br /><img')
        movieInfo = re.findall(pattern, movieContent[0])

        if (len(movieInfo) > 0):
            movieInfo = movieInfo[0]+''

            # 刪除<br />標簽
            movieInfo = movieInfo.replace("<br />","")

            # 根據 ◎ 符號拆分

            movieInfo = movieInfo.split('◎')

        else:
            movieInfo = ""

        print("電影基礎信息: ")

        for item in movieInfo:
            print(item)


        # 電影海報
        pattern = re.compile('<img.*? src="(.*?)".*? />', re.S)        
        movieImg = re.findall(pattern,movieContent[0])

        if (len(movieImg) > 0):
            movieImg = movieImg[0]
        else:
            movieImg = ""
 
        print("電影海報: " + movieImg)

        pattern = re.compile('<td style="WORD-WRAP: break-word" bgcolor="#fdfddf"><a href="(.*?)">.*?</a></td>', re.S)
        movieDownUrl = re.findall(pattern,movieContent[0])

        if (len(movieDownUrl) > 0):
            movieDownUrl = movieDownUrl[0]
        else:
            movieDownUrl = ""

        print("電影下載地址：" + movieDownUrl + "")

        print("------------------------------------------------\n\n\n")

執行抓取

if __name__=='__main__':


     print("開始抓取電影數據");
     
     queryMovieList()

     print(len(movieUrls))


     queryMovieInfo(movieUrls)

     print("結束抓取電影數據")