scrapy 圖片爬取 多層多頁 保存不同的文件夾 重命名full文件夾

記錄下整個爬蟲代碼，我已經把實驗網站爬完了。。

items.py

import scrapy


class DemoItem(scrapy.Item):
    # define the fields for your item here like:
    folder_name = scrapy.Field()  #選取頁面里的主題或者標題作為文件夾名字，代替full文件夾
    #img_name = scrapy.Field()  # 提取的圖片的名字，如果沒有就不使用
    img_url = scrapy.Field()  # 圖片的鏈接

spider.py

# -*- coding: utf-8 -*-
import scrapy
from demo.items import DemoItem


class LogosSpider(scrapy.Spider):
    name = 'logos'
    allowed_domains = ['tttt8.net']
    #start_urls = ['http://www.tttt8.net/category/legbaby/']
    #start_urls = ['http://www.tttt8.net/category/ugirls/']
    #start_urls = ['http://www.tttt8.net/category/kelagirls/']
    start_urls = ['http://www.tttt8.net/category/xiurenwang/micatruisg/']
    #這里的page是用來構造一級頁面翻頁鏈接
    page = 1

    def parse(self, response):
        #提取所有專欄的列表
        li_list = response.xpath('//*[@id="post_container"]/li')
        for li in li_list:
            #實例化
            item = DemoItem()
            #這里只提取專欄的名字，用於后面儲存文件夾的命名
            item['folder_name'] = li.xpath('./div[2]/h2/a/text()').extract_first()
            #提取二級頁面的鏈接，准備給二級頁面函數去工作
            next_plink = li.xpath('./div[1]/a/@href').extract_first()
            #item把一級頁面實例化的內容傳送給二級頁面接收
            yield scrapy.Request(url = next_plink, callback = self.parse2, meta = {'item':item})
        # 一級頁面的翻頁列表/ 自行查找別的方法翻頁，網上好多，我覺得這個構造更簡單
        page_list = response.xpath('//div[@class="pagination"]/a/@href').extract()
        #找到最后一頁的頁碼
        last_page = page_list[-1]
        #提取出最大的頁碼
        max_num = int(last_page[-2])
        #構造翻頁的頁碼鏈接
        if self.page <= max_num:
            self.page += 1
            new_page_url = self.start_urls[0] + 'page/' + str(self.page) + '/'
            yield scrapy.Request(url = new_page_url, callback = self.parse)

    def parse2(self, response):
        #接收一級頁面的內容，和翻頁請求的銜接，不然翻頁就丟失了，會報錯
        item = response.meta['item']
        p_list = response.xpath('//*[@id="post_content"]/p/img')
        #正常的提取圖片的鏈接給pepeline去下載
        for img in p_list:
            img_url = img.xpath('./@src').extract_first()
            # 這里必須加中括號[]，圖片下載的函數，要求是list類型
            item['img_url'] = [img_url]
            yield item
        # 二級頁面的翻頁列表，然后在后面yield請求
        next_page_list = response.xpath('//*[@id="content"]/div[1]/div[3]/div[2]/a/@href').extract()
        for next_page in next_page_list:
            #這里一定要加meta，不然二級的翻頁就error，這里我搞了好久才發現。
            yield scrapy.Request(url = next_page, callback = self.parse2, meta = {'item':item})

settings.py

# -*- coding: utf-8 -*-


BOT_NAME = 'demo'
SPIDER_MODULES = ['demo.spiders']
NEWSPIDER_MODULE = 'demo.spiders'

#存儲路徑和header
IMAGES_STORE = 'D:\pics'
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'

DOWNLOAD_DELAY = 0.2
#機器人關掉
ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
    'demo.pipelines.DemoPipeline': 300,
}
#下載指定的field
IMAGES_URLS_FIELD = 'img_url'

pipelines.py

import scrapy
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline


class DemoPipeline(ImagesPipeline):
    # 固定的改寫的函數，不需要修改
    def get_media_requests(self, item, info):
        for img_url in item['img_url']:
            # 下載完后給別的函數去改名字，所以用meta傳下去
            yield scrapy.Request(img_url, meta = {'item':item})

    def file_path(self, request, response = None, info = None):
        item = request.meta['item']
        folder_name = item['folder_name']
        # img_name = item['img_name']  #圖片沒有名字不啟用這個語句
        # 因為圖片沒有名字就用url截取最后的字符串作為名字
        image_guid = request.url.split('/')[-1]
        img_name = image_guid
        # name = img_name + image_guid
        # name = name + '.jpg'
        # 0 代表文件夾，1 代表文件
        filename = u'{0}/{1}'.format(folder_name, img_name)
        return filename

    # 固定改寫的函數，不需要修改
    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem('Image Downloaded Failed')
        return item

結果：