scrapy中使用代理

tip:

大致思路：從網絡（URL：http://ip.jiangxianli.com/api/proxy_ips）獲取代理ip以及端口，存到列表；隨機一個數從列表中取一個ip，設置超市時間以及次數；捕獲異常，從列表中刪除無用的ip代理並設置新的ip代理

settings.py中還要開啟下載中間件

DOWNLOADER_MIDDLEWARES = {
   'tianmao.middlewares.TestDownloaderMiddleware': 543,
}

code：

from scrapy import signals
import requests
import json, random


class TestDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    def __init__(self):
        # 獲取請求到的ip內容
        res = requests.get('http://ip.jiangxianli.com/api/proxy_ips')
        # 存入列表
        self.ip_list = json.loads(res.content)['data']['data']
        self.random_int = 1
        print('init method is running ...')


    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called

        # print(request.meta)

        print('process request is running ...')

        # 獲取代理
        self.get_proxy(request)

        return None

    def process_response(self, request, response, spider):

        print('process_response is running ...')

        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        print('exception is %s' % exception)

        if exception:
# 再次選一個IP地址，返回request
            request = self.get_proxy(request)

        return request

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    def get_proxy(self, request):
        num_ip = len(self.ip_list)
        print('現在總共有%d個ip地址' % num_ip)

        # 隨機一個ip
        self.random_int = random.randint(0, num_ip)
        print('隨機到的整數是%d' % self.random_int)
        ip_dic = self.ip_list[self.random_int+1]
        print('隨機到的ip地址是：%s' % ip_dic)

        ip = ip_dic.get('ip')
        port = ip_dic.get('port')

        ip_address = 'http://' + ip + ':' + port

        # 設置代理
        request.meta['proxy'] = ip_address

        # 設置最大請求時間
        request.meta['download_timeout'] = 5

        return request