Scrapy之下載中間件與爬蟲中間件

執行流程

　　1.引擎找到要執行的爬蟲，並執行爬蟲的start_requests方法，並得到一個迭代器　

　　2.迭代器循環時候會獲取到Request對象，而request對象中封裝了要訪問的url和回調函數

　　3.將所有的request對象(任務)放到調度器中，用於以后被下載器下載

　　4.下載器去調度器中獲取要下載任務（就是Request對象），下載完成后執行回調函數

　　5.回到spider的回調函數中

　　　　yield  Resquest()

　　　　yield  Item()

下載中間件(之前代理就是下載中間件做的)　　　

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from scrapy.http import HtmlResponse
from scrapy.http import Request


class Md1(object):

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        print('md1.process_request', request)
        # return None  # 默認返回None
        # 1.返回Response對象
        # 如果有返回值md2的process_request都不執行了，直接跳到最后一個中間件的process_response方法一層層的返回，
        # 而在django中，它會從當前中間件的process_response直接返回(這點和Django有點大不相同)
        # 而且中間件我直接截獲了 偽造之前的url，我自己去www.xxx.com下載，下載的結果body
        # url = request.url
        # return HtmlResponse(url='www.xxx.com', status=200, headers=None, body=b'innerjob')
        # import requests
        # result = requests.get(request.url)
        # return HtmlResponse(url=request.url, status=200, headers=None, body=result.content)

        # 2.返回Request對象  這樣的話就會在一直轉圈，來回跑到調度器
        # return Request(url='https://dig.chouti.com/r/tec/hot/1')

        # 3.拋出異常  就必須寫上process_exception()方法 不然報錯
        # from scrapy.exceptions import IgnoreRequest
        # raise IgnoreRequest

        # 4.一般是對請求加工  但是這個功能內置有了 加cookie也都可以各種加東西內置也有了cookie
        request.headers['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        print('md1.process_response', request, response)
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass


class Md2(object):

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        print('md2.process_request', request)
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        print('md2.process_response', request, response)
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

應用

DOWNLOADER_MIDDLEWARES = {
   # 'redisdepth.middlewares.RedisdepthDownloaderMiddleware': 543,
   #  'redisdepth.md.Md1': 666,
   #  'redisdepth.md.Md2': 667
}

流程圖：

 

 爬蟲中間件（之前深度就是這樣子做的）

#!/usr/bin/env python
# -*- coding:utf-8 -*-


class Sd1(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        # crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) #這是關於擴展需要用的先不看 信號，在Django里邊也有信號這個東西
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        # 下載完成之后執行,然后交給parse處理
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        # spider處理完成
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request, dict
        # or Item objects.
        # 異常調用
        # return: None,繼續交給后續中間件處理異常；含 Response 或 Item 的可迭代對象(iterable)，交給調度器或pipeline
        pass

    # 只在爬蟲啟動時候，只執行一次 讀取最開始爬蟲start_requests方法中返回生成器的 然后循環在這兒一個個的返回
    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class Sd2(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        # crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) #這是關於擴展需要用的先不看 信號，在Django里邊也有信號這個東西
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

 應用

SPIDER_MIDDLEWARES = {
   # 'redisdepth.middlewares.RedisdepthSpiderMiddleware': 543,
    'redisdepth.sd.Sd1': 666,
    'redisdepth.sd.Sd2': 667,

}

 流程圖

 

 內置的下載中間件

　　

 內置的爬蟲中間件

這是做實驗的截圖