爬蟲———通過pipeline以及items 將數據以json類型存儲

1、books.py

import scrapy
from scrapy.selector.unified import SelectorList

class BooksSpider(scrapy.Spider):
    name = 'books'
    allowed_domains = ['books.toscrape.com']
    start_urls = ['http://books.toscrape.com/catalogue/page-1.html']

    def parse(self, response):
        # print('*')
        # print(type(response))    #  <class 'scrapy.http.response.html.HtmlResponse'>
        # print('*')

        Books_lis = response.xpath("//div/ol[@class='row']/li")
        # print('*')
        # print(type(Books_lis))
        # print('*')
        for Books_li in Books_lis:
            book_name = Books_li.xpath(".//h3/a/@title").get()
            book_price = Books_li.xpath(".//div[@class='product_price']/p[@class='price_color']/text()").getall()

            book_price = "".join(book_price)
            books_all = {"book_name":book_name,"book_price":book_price}
            yield books_all

2、pipeline .py

import json


class BooksPipeline(object):
    def __init__(self):
        self.file = open("books_all.json", "w", encoding="utf-8")

    def open_spider(self, spider):
        # self.file = open('items.jl', 'w')
        print("爬蟲開始了。。。")

    def close_spider(self, spider):
        self.file.close()
        print("爬蟲結束了。。。")

    def process_item(self, item, spider):
        # line = json.dumps(dict(item)) + "\n"
        # self.file.write(line)
        item_json = json.dumps(item,ensure_ascii= False)
        self.file.write(item_json+'\n')
        return item

3、結果

 

 

 

 

 3、犯的錯誤

 

 

 

 

 

 

 4、專業化

設置item.py文件

from scrapy import Item,Field


class BooksItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    book_name = Field()
    book_price = Field()