selenium的使用技巧及集成到scrapy

為了爬取拉鈎,今天學習了selenum的使用技巧.

 

from scrapy.http import HtmlResponse

 

class JSPageMiddleware(object):

 

    def process_request(self, request, spider):

        if spider.name == "zhihu":

            browser = webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")

            browser.get(request.url)

            time.sleep(3)

            print("訪問：{0}".format(request.url))

            # 請求已完成，所以不用再發送到下載器。用HtmlResponse 之后就不會發送到downlaoder,而是直接返回給spider

            return HtmlResponse(url=browser.current_url, body=browser.page_source, encoding="utf-8", request=request)

 

這樣有個不好的地方就是每次來一個請求時都會啟動一次瀏覽器,瀏覽器啟動是很慢的,可以在類中初始化一個瀏覽器,

class JSPageMiddleware(object):

    def __init__(self):

        self.browser=webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")

        super(JSPageMiddleware,self).__init__()

 

    def process_request(self, request, spider):

        if spider.name == "zhihu":

            self.browser.get(request.url)

            time.sleep(3)

            print("訪問：{0}".format(request.url))

            # 請求已完成，所以不用再發送到下載器。用HtmlResponse 之后就不會發送到downlaoder,而是直接返回給spider

            return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8", request=request)

 

用上述方法,爬蟲自動關閉時,瀏覽器不會關閉,並且每個spider會共用一個打開的瀏覽器,這樣也不方便調試,可以把這個初始化的工作放在各個spider中

# -*- coding: utf-8 -*-

import scrapy

from selenium import webdriver

from scrapy import signals

from scrapy.xlib.pydispatch import dispatcher

from scrapy.http import HtmlResponse

 

 

class ZhihuSpider(scrapy.Spider):

    name = 'zhihu'

    allowed_domains = ['oschina.net/blog']

    start_urls = ['https://www.oschina.net/blog']

 

    def __init__(self):

        self.browser = webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")

        super(ZhihuSpider, self).__init__()

#傳遞信息,也就是當爬蟲關閉時scrapy會發出一個spider_closed的信息,當這個信號發出時就調用closeSpider函數關閉這個瀏覽器.

        dispatcher.connect(self.closeSpider, signals.spider_closed)

 

    def closeSpider(self, spider):

        print("spider closed")

        # 當爬蟲退出的時關閉瀏覽器

        self.browser.quit()

 

    def parse(self, response):

        # data=response.css(".SignFlow-accountInpu input[]").extract()

        pass

 

Middleware 中的類

class JSPageMiddleware(object):

    def process_request(self, request, spider):

        if spider.name == "zhihu":

            spider.browser.get(request.url)

            time.sleep(3)

            print("訪問：{0}".format(request.url))

            return HtmlResponse(url=spider.browser.current_url, body=spider.browser.page_source, encoding="utf-8",request=request)

 

注意 上面三種方法都必須在settings中把JSPageMiddleware加入

DOWNLOADER_MIDDLEWARES = {

# 'outlook.middlewares.MyCustomDownloaderMiddleware': 543,

'outlook.middlewares.JSPageMiddleware': 1,