處理scrapy中包括多個pipeline時如何讓spider執行制定的pipeline管道
１:創建一個裝飾器
from scrapy.exceptions import DropItem
import functools
'''
當有多個pipeline時,判斷spider如何執行指定的管道
'''
def check_spider_pipeline(process_item_method):
@functools.wraps(process_item_method)
def wrapper(self, item, spider):
# message template for debugging
msg = '%%s %s pipeline step' % (self.__class__.__name__,)
if self.__class__ in spider.pipeline:#判斷要執行的spider中是否包含所需的pipeline 如果有則執行否則拋出DropItem信息
spider.logger.debug(msg % 'executing')
return process_item_method(self,item,spider)
# otherwise, just return the untouched item (skip this step in
# the pipeline)
else:
spider.logger.debug(msg % 'skipping')
raise DropItem("Missing pipeline property")
return wrapper
2:在每個spider所在的類中添加一個pipeline數組，里面包含要執行的pipeline的名字
-*- coding: utf-8 -*-
from scrapy.spiders import CrawlSpider,Rule
# from scrapy.selector import Selector
from ..items import BotcnblogsItem,BotItem
from scrapy.linkextractors import LinkExtractor
import re
from ..BotcnblogsPipeline import BotcnblogsPipeline
class CnblogsSpider(CrawlSpider):
pipeline = set([BotcnblogsPipeline,])
#爬蟲名稱
name = "cnblogs"
#設置允許的域名
allowed_domains = ["cnblogs.com"]
#設置開始爬去的頁面
start_urls = (
'http://www.cnblogs.com/fengzheng/',
)
rules=(
Rule(LinkExtractor(allow=('fengzheng/default.html\?page\=([\d]+)')),callback='parse_item',follow=True),
# Rule(LinkExtractor(allow=('fengzheng/p/([\d]+).html')),callback='parse_info',follow=True),
)
3:在要執行的pipeline中的process_item方法加上裝飾器，這樣就可以過濾pipeline了
import json
from .checkpipe import check_spider_pipeline
class BotcnblogsPipeline(object):
def __init__(self):
self.file=open('jd.json','w+')
@check_spider_pipeline
def process_item(self,item,spider):
#此處如果有中文的話，要加上ensure_ascii=False參數，否則可能出現亂碼
record=json.dumps(dict(item),ensure_ascii=False)+"\n"
self.file.write(record)
return item
def open_spider(self,spider):
print("打開爬蟲了")
def close_spider(self,spider):
print("關閉爬蟲")
self.file.close()
具體例子可以參考其中的cnblogs spider的例子 下載