深入爬蟲書scrapy 之json內容沒有寫入文本


settings.py設置

ITEM_PIPELINES = {
   'tets.pipelines.TetsPipeline': 300,
}
View Code

spider代碼

xpath后綴添加.extract() parse()返回return item

import scrapy
from tets.items import TetsItem

class KugouSpider(scrapy.Spider):
    name = 'kugou'
    allowed_domains = ['www.kugou.com']
    start_urls = ['http://www.kugou.com/']

    def parse(self, response):
        item = TetsItem()
        item['title'] = response.xpath("/html/head/title/text()").extract()
        print(item['title'])
        return item
View Code

piplines代碼

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json

class TetsPipeline(object):
    def __init__(self):
        # self.file = codecs.open("D:/git/learn_scray/day11/mydata2.txt", "wb", encoding="utf-8")
        self.file = codecs.open("D:/git/learn_scray/day11/1.json", "wb", encoding="utf-8")

    # 處理文本(xx.txt)
    # def process_item(self, item, spider):
    #     l = str(item) + "\n"
    #     print(l)
    #     self.file.write(l)
    #     return item
    def process_item(self, item, spider):
        print("進入")
        # print(item)
        i = json.dumps(dict(item), ensure_ascii=False)
        # print("進入json")
        # print(i)
        l = i + "\n"
        print(l)
        self.file.write(l)
        return item

    def close_spider(self, spider):
        slef.file.close()
View Code

items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TetsItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
View Code

結果如圖下

 

 

 


免責聲明!

本站轉載的文章為個人學習借鑒使用,本站對版權不負任何法律責任。如果侵犯了您的隱私權益,請聯系本站郵箱yoyou2525@163.com刪除。



 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM