scrapy從命令行傳值

1.新建文件run.py

from scrapy.cmdline import execute


# tmall：爬蟲的名字
# pro=男裝為需要傳入的參數值
execute(['scrapy', 'crawl', 'tmall', '-a', 'pro=男裝', '--nolog'])

# 下面是無參數用法
# execute(['scrapy', 'crawl', 'tmall', '--nolog'])

 

2.爬蟲.py中重寫init方法，傳入參數；（字典編碼以字符串形式拼接到url后邊）

# 倒入頭文件 （字典編碼后以參數形式拼接到url）
from urllib.parse import urlencode


class TmallSpider(scrapy.Spider):
    name = 'tmall'
    allowed_domains = ['tmall.com']

    def __init__(self, pro=None, *args, **kwargs):
        super(TmallSpider, self).__init__(*args, **kwargs)
        self.params = {
            'q': pro,
            'total_Page': 1,
            'jumpto': 1,
        }

        self.start_url = 'https://list.tmall.com/search_product.htm?' + urlencode(self.params)

    def start_requests(self):
        print('self.start_url:' + self.start_url)
        # yield scrapy.Request(
        #     url=self.start_url,
        #     callback=self.get_total_page,
        #     dont_filter=True,
        # )

    def get_total_page(self, response):
        pass