對空氣質量歷史數據的爬取

爬取https://www.aqistudy.cn/historydata 網站的空氣質量報告，爬取的數據以csv文件生成

scrapy startproject air_quality 創建scrapy項目

scrapy genspider api_history_spider https://www.apistudy.cn/historydata/index.php 編寫spider

文件目錄如圖所示

seetings.py

ITEM_PIPELINES = {
    'air_quality.pipelines.AirQualityPipeline': 300,
}

items.py

import scrapy


class AirQualityItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    city_name = scrapy.Field()  # 城市名稱
    record_date = scrapy.Field()  # 檢測日期
    aqi_val = scrapy.Field()  # AQI
    range_val = scrapy.Field()  # 范圍
    quality_level = scrapy.Field()  # 質量等級
    pm2_5_val = scrapy.Field()  # PM2.5
    pm10_val = scrapy.Field()  # PM10
    so2_val = scrapy.Field()  # SO2
    co_val = scrapy.Field()  # CO
    no2_val = scrapy.Field()  # NO2
    o3_val = scrapy.Field()  # O3
    rank = scrapy.Field()  # 排名

pipelines.py

from scrapy.exporters import CsvItemExporter

class AirQualityPipeline(object):

    def open_spider(self,spider):
        self.file = open('air_quality.csv', 'wb')
        self.exporter = CsvItemExporter(self.file)
        self.exporter.start_exporting()

    def close_spider(self,spider):
        self.exporter.finish_exporting()
        self.file.close()

    def process_item(self, item,spider):
        self.exporter.export_item(item)
        return item

api_history_spider.py

# -*- coding: utf-8 -*-
import scrapy
from urllib import parse
from air_quality.items import AirQualityItem

base_url = 'https://www.aqistudy.cn/historydata/'

class ApiHistorySpiderSpider(scrapy.Spider):
    name = 'api_history_spider'
    allowed_domains = ["aqistudy.cn"]
    start_urls = ['https://www.aqistudy.cn/historydata/']

    def parse(self, response):
        """
            解析初始頁面
        """
        # 獲取所有城市的URL
        city_url_list = response.xpath('//div[@class="all"]//div[@class="bottom"]//a//@href')

        for city_url in city_url_list:
            # 依次遍歷城市URL
            city_month_url = base_url + city_url.extract()
            # 解析每個城市的月份數據
            request = scrapy.Request(city_month_url, callback=self.parse_city_month)
            yield request

    def parse_city_month(self, response):
        """
            解析該城市的月份數據
        """
        # 獲取該城市的所有月份URL
        month_url_list = response.xpath('//table[@class="table table-condensed '
                                        'table-bordered table-striped table-hover '
                                        'table-responsive"]//a//@href')

        for month_url in month_url_list:
            # 依次遍歷月份URL
            city_day_url = base_url + month_url.extract()
            # 解析該城市的每日數據
            request = scrapy.Request(city_day_url, callback=self.parse_city_day)
            yield request

    def parse_city_day(self, response):
        """
            解析該城市的每日數據
        """
        url = response.url
        item = AirQualityItem()
        city_url_name = url[url.find('=') + 1:url.find('&')]

        # 解析url中文
        # item['city_name'] = city_url_name
        item['city_name'] = parse.unquote(city_url_name)

        # 獲取每日記錄
        day_record_list = response.xpath('//table[@class="table table-condensed '
                                         'table-bordered table-striped table-hover '
                                         'table-responsive"]//tr')
        for i, day_record in enumerate(day_record_list):
            if i == 0:
                # 跳過表頭
                continue
            td_list = day_record.xpath('.//td')

            item['record_date'] = td_list[0].xpath('text()').extract_first()  # 檢測日期
            item['aqi_val'] = td_list[1].xpath('text()').extract_first()  # AQI
            item['range_val'] = td_list[2].xpath('text()').extract_first()  # 范圍
            item['quality_level'] = td_list[3].xpath('.//div/text()').extract_first()  # 質量等級
            item['pm2_5_val'] = td_list[4].xpath('text()').extract_first()  # PM2.5
            item['pm10_val'] = td_list[5].xpath('text()').extract_first()  # PM10
            item['so2_val'] = td_list[6].xpath('text()').extract_first()  # SO2
            item['co_val'] = td_list[7].xpath('text()').extract_first()  # CO
            item['no2_val'] = td_list[8].xpath('text()').extract_first()  # NO2
            item['o3_val'] = td_list[9].xpath('text()').extract_first()  # O3
            item['rank'] = td_list[10].xpath('text()').extract_first()  # 排名

            yield item

運行spider

scrapy crawl api_history_spider

得到csv文件，部分如下圖所示：