scrapy數據增量式爬取

爬取過的數據跳過
1、通過url判斷
2、通過數據指紋判斷

 

創建爬蟲項目 ：scrapy startproject  xxx

cd xxx

創建爬蟲文件：scrapy genspider -t crawl spidername www.xxx.com

一、根據url判斷

爬蟲文件

# -*- coding: utf-8 -*-
import scrapy
from redis import Redis
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from increment1.items import Increment1Item
"""
爬取過的數據跳過
1、通過url判斷
2、通過數據指紋判斷
"""


class FirstSpider(CrawlSpider):
    name = 'first'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.4567tv.tv/index.php/vod/show/id/7.html']

    rules = (
        Rule(LinkExtractor(allow=r'/index.php/vod/show/id/7/page/\d+\.html'), callback='parse_item', follow=True),
    )

    def parse_detail(self, response):
        item = response['item']
        actor = response.xpath('//div[@class="stui-content__detail"]/p[3]//text()').extract_first()
        item['actor'] = actor
        yield item

    def parse_item(self, response):
       
        conn = Redis(host='127.0.0.1', port=6388)
        detail_url_list = "https://www.4567tv.tv" + response.xpath('//li[@class="col-md-6 col-sm-4 col-xs-3"]/div/a/@href').extract_first()
        for url in detail_url_list:
            item = Increment1Item()
            ex = conn.sadd('movies_url', url)
            if ex == 1:   # 說明redis里沒有該url
                yield scrapy.Request(url=url, callback=self.parse_detail, meta={"item", item})
            else:
                print('爬過的數據不在爬取！！！')

 

在管道文件里進行存儲

from redis import Redis


class Increment1Pipeline(object):
    def open_spider(self, spider):
        self.conn = Redis(host='127.0.0.1', port=6388)

    def process_item(self, item, spider):
        print('新數據寫入')
        dic = {
            'actor': item['actor']
        }
        self.conn.lpush('move_data', dic)

 

二、根據數據進行指紋識別

爬蟲文件

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from incerment2.items import Incerment2Item
import hashlib
from redis import Redis


class FirstSpider(CrawlSpider):
    name = 'first'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    rules = (
        Rule(LinkExtractor(allow=r'/text/page/\d+/'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        self.conn = Redis(host='127.0.0.1', port=6388)

        div_list = response.xpath('//div[@class="content-left"]/div')
        for div in div_list:
            item = Incerment2Item()
            item['author'] = div.xpath('./div[1]/a[2]/h2/text()').extract_first()
            content = div.xpath('./a[1]/div/span/text()').extract()
            content = ''.join(content)
            item['content'] = content

            # 將當前爬取的數據做哈希唯一標識(數據指紋)
            source = item['author']+item['content']
            hashValue = hashlib.sha256(source.encode()).hexdigest()

            ex = self.conn.sadd('hashValue', hashValue)
            if ex == 1:
                yield item

            else:
                print('數據未更新')