Python的scrapy之爬取妹子圖片

閑來無事，做的一個小爬蟲項目

爬蟲主程序：

import scrapy
from ..items import MeiziItem

class MztSpider(scrapy.Spider):
    name = 'mzt'
    allowed_domains = ['meizitu.com']
    start_urls = ['http://meizitu.com/']

    def parse(self, response):
        tags = response.xpath(".//*[@class='tags']/span/a")
        for i in tags:
            item = MeiziItem()
            tag_href = i.xpath(".//@href").extract()[0]
            tag_name = i.xpath(".//@title").extract()[0]
            item['tag_name'] = tag_name
            item['tag_href'] = tag_href
            #print(item['tag_name'])
            #yield item
            yield scrapy.Request(url=item['tag_href'], meta={'item': item}, callback=self.parse_page)

    def parse_page(self, response):

        item = response.meta['item']
        # 進入某個標簽后，爬取底部分頁按鈕
        page_lists = response.xpath(".//*[@id='wp_page_numbers']/ul/li")
        # 獲取底部分頁按鈕上的文字，根據文字來判斷當前標簽頁下總共有多少分頁
        page_list = page_lists.xpath('.//text()')
        # 如果當前標簽頁下有多個頁面，則再根據第一個按鈕是否為“首頁”來進行再次提取，因為這里有的頁面第一個按鈕是首頁，有的第一個按鈕是“1”
        if len(page_lists) > 0:
            if page_list[0].extract() == '首頁':
                page_num = len(page_lists) - 3
            else:
                page_num = len(page_lists) - 2
        else:
            page_num = 1

        # 根據當前標簽頁的鏈接，來拼成帶頁碼的鏈接
        if '_' in item['tag_href']:
            index = item['tag_href'][::-1].index('_')
            href_pre = item['tag_href'][:-index]
        else:
            if page_num == 1:
                href_pre = item['tag_href'].split('.html')[0]
            else:
                href_pre = item['tag_href'].split('.html')[0] + '_'
        for i in range(1, page_num + 1):
            item = response.meta['item']
            if page_num == 1:
                href = href_pre + '.html'
            else:
                href = href_pre + str(i) + '.html'
            item['page_list'] = href
            #yield item
            yield scrapy.Request(url=item['page_list'], meta={'item': item}, callback=self.parse_album)

    def parse_album(self, response):
        albums = response.xpath(".//*[@class='pic']")
        for album in albums:
            item = response.meta['item']
            album_href = album.xpath(".//a/@href").extract()[0]
            album_name = album.xpath(".//a/img/@alt").extract()[0]
            item['album_name'] = album_name
            item['album_href'] = album_href
            #yield item
            yield scrapy.Request(url=item['album_href'], meta={'item': item}, callback=self.parse_img)

    def parse_img(self, response):
        img_list = response.xpath(".//*/p/img")
        for img in img_list:
            item = response.meta['item']
            img_title = img.xpath(".//@alt").extract()[0]
            if img_title == '':
                for i in range(1, len(img_list + 1)):
                    img_title = item['album_name'] + '_' + str(i)
            else:
                img_title = img_title
            img_urls = img.xpath(".//@src").extract()
            img_src = img.xpath(".//@src").extract()[0]
            item['img_title'] = img_title
            item['img_src'] = img_src
            item['img_urls'] = img_urls
            yield item

items設置

import scrapy


class MeiziItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 標簽名稱
    tag_name = scrapy.Field()
    # 標簽鏈接
    tag_href = scrapy.Field()
    # 進入某標簽后的所有鏈接，加頁碼的
    page_list = scrapy.Field()
    # 圖片專輯名稱
    album_name = scrapy.Field()
    # 圖片專輯鏈接
    album_href = scrapy.Field()
    # 照片標題
    img_title = scrapy.Field()
    # 照片鏈接
    img_src = scrapy.Field()
    # 照片鏈接集合，用於ImagesPipeline下載圖片
    img_urls = scrapy.Field()

輸出管道：

print('正在爬取...')
print('老濕機，請耐心等待喲...')
class MeiziPipeline(object):
    def process_item(self, item, spider):
        print('標簽名稱:',item['tag_name'])
        print('標簽鏈接:',item['tag_href'])
        print('頁碼:',item['page_list'])
        print('圖片專輯名稱:',item['album_name'])
        print('圖片專輯鏈接:',item['album_href'])
        print('照片標題:',item['img_title'])
        print('照片鏈接:',item['img_src'])
        print('照片鏈接集合:',item['img_urls'])
        print('----------------')
        return item

保存到本地的管道：

import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem


class MztImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for image_url in item['img_urls']:
            yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("該Item沒有圖片")
        return item

setting的設置，往setting.py加入

BOT_NAME = 'meizi'

SPIDER_MODULES = ['meizi.spiders']
NEWSPIDER_MODULE = 'meizi.spiders'

IMAGES_STORE = r'G:\\mzt'    # 圖片存儲路徑
IMAGES_EXPIRES = 90             # 過期天數
IMAGES_MIN_HEIGHT = 100         # 圖片的最小高度
IMAGES_MIN_WIDTH = 100          # 圖片的最小寬度

爬取的最終結果

本來想把這些圖片分門分類的保存，然而不太會，所有的圖片全保存在一個文件夾下面，，，