Scrapy 實現爬取多頁數據 多層url數據爬取 meta傳遞 鏈接為相對路徑

提取電影網站的片名，導演，影片播放地址。

item.py

import scrapy


class MovieItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field() #沒啥好說的，定義三個返回變量
    actor = scrapy.Field()
    link = scrapy.Field()

spider,py

import scrapy

from movie.items import MovieItem

class MovieproSpider(scrapy.Spider):
    name = 'moviePro'
    allowed_domains = ['4567tv.tv']
    start_urls = ['https://www.4567tv.tv/frim/index1.html']
    page = 1
    page_url = 'https://www.4567tv.tv/frim/index1-%s.html'


    def parse(self, response):
        li_list = response.xpath('//li[@class="col-md-6 col-sm-4 col-xs-3"]')
        for li in li_list:
            item = MovieItem()
            name = li.xpath('./div/div/h4/a/text()').extract_first()
            detail_url = 'https://www.4567tv.tv' + li.xpath('./div/div/h4/a/@href').extract_first()
            item['name'] = name
            yield scrapy.Request(url = detail_url, callback = self.parse_detail, meta ={'item':item})

        if self.page <= 10:
            self.page += 1
            new_page_url = self.page_url % self.page
            yield scrapy.Request(url = new_page_url, callback = (self.parse))

    def parse_detail(self, response):
        item = response.meta['item'] #注意這里實例化的是meta的，是parse函數傳遞過來的第二層內容
        actor = response.xpath('/html/body/div[1]/div/div/div/div[2]/p[3]/a/text()').extract_first() 
        link = response.xpath('/html/body/div[1]/div/div/div/div[1]/a/@href').extract_first()
        item['actor'] = actor
        item['link'] = 'https://www.4567tv.tv' + link #這個連接是相對路徑，重新構造地址
        yield item

settings.py

 FEED_EXPORT_ENCODING ='utf-8' #插入這兩行，解碼成中文，不然出現的是亂碼

2 FEED_EXPORT_ENCODING = 'gb18030' 

運行

scrapy crawl moviePro -o mov.csv

結果