解析json文件，json接口，將內容入數據庫（Python腳本）

剛接觸用Python解析數據，發現十分的方便

 

先來了解一下json.loads()、json.dumps()和json.dump()、json.load()四個函數

json.loads()、json.dumps()和json.dump()、json.load()分別是兩組不一樣用法

帶s的用於數據類型的轉換，不帶s的用於操做文件。

json.loads()、json.dumps()概念理解

json自己是字符串，經過json.loads()、json.dumps()能夠進行字典和字符串的轉換。

由於瀏覽器不支持字典方式顯示，若是請求過來的類型是字典，必須經過json.dumps()函數將字典轉換為字符串以后，才可展現。

json.dump()和json.load()概念理解

主要用來讀寫json文件函數

 

 

1.通過接口解析json數據提取各個城市的實時疫情數據，沒有在腳本里進行定時，不知道該怎么寫定時任務（查百度也弄不明白），就在linux系統設置了，一天執行一次腳本

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import json
import time
from pymysql import *

def mes():
    url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0'  #請求地址
    headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.6181'}#創建頭部信息
    resp =  requests.get(url,headers = headers)  #發送網絡請求
    content=resp.content.decode('utf-8')
    print("content"+content)
    soup = BeautifulSoup(content, 'html.parser')
    print(soup)
    listA = soup.find_all(name='script',attrs={"id":"getAreaStat"})

    account =str(listA)

    mes = account.replace('[<script id="getAreaStat">try { window.getAreaStat = ', '')

    mes=mes.replace('}catch(e){}</script>]','')
    #mes=account[52:-21]
    #json.loads 用於解碼 JSON 數據。該函數返回 Python 字段的數據類型。

    messages_json = json.loads(mes)
    print("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
    print(messages_json)
    times=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
    print(times)
    provinceList=[]
    cityList=[]
    lenth=total()
    total()
    sheng = len(messages_json)
    con=sheng+(lenth)#算出數據庫已有的條數+今天省份的條數，才是城市的開始id
    print(sheng)
    print(len(messages_json))
    print(con)
    for item in messages_json:
        lenth+=1
        provinceName=item['provinceName']
        nowCount = item['currentConfirmedCount']
        confirmedCount=item['confirmedCount']
        suspectedCount=item['suspectedCount']
        curedCount=item['curedCount']
        deadCount=item['deadCount']
        cities=item['cities']
        provinceList.append((lenth,times,provinceName,None,nowCount,confirmedCount,suspectedCount,curedCount,deadCount))
        for i in cities:
            con+=1
            provinceName = item['provinceName']
            cityName=i['cityName']
            nowCount = i['currentConfirmedCount']
            confirmedCount = i['confirmedCount']
            suspectedCount = item['suspectedCount']
            curedCount = i['curedCount']
            deadCount = i['deadCount']
            cityList.append((con,times,provinceName,cityName,nowCount,confirmedCount,suspectedCount,curedCount,deadCount))
    insert(provinceList,cityList)
    return NULL


def insert(provinceList, cityList):
    provinceTuple=tuple(provinceList)
    cityTuple=tuple(cityList)
    cursor = db.cursor()
    sql = "insert into China values (%s,%s,%s,%s,%s,%s,%s,%s,%s) "
    try:
        cursor.executemany(sql,provinceTuple)
        print("插入成功")
        db.commit()
    except Exception as e:
        print(e)
        db.rollback()
    try:
        cursor.executemany(sql,cityTuple)
        print("插入成功")
        db.commit()
    except Exception as e:
        print(e)
        db.rollback()
    cursor.close()
def total():
    sql= "select * from China"
    cursor = db.cursor()
    try:
        cursor.execute(sql)
        results = cursor.fetchall()
        lenth = len(results)
        db.commit()
        return lenth
    except:
        print('執行失敗，進入回調1')
        db.rollback()

# 連接數據庫的方法
def connectDB():
    try:
        db = connect(host='localhost', port=3306, user='root', password='123456', db='db2',charset='utf8')
        print("數據庫連接成功")
        return db
    except Exception as e:
        print(e)
    return NULL
if __name__ == '__main__':
    db=connectDB()
    mes()

 

sql:

CREATE TABLE `china`  (
  `id` int(11) NOT NULL,
  `time` datetime NULL DEFAULT NULL,
  `provinceName` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '省，直轄市',
  `cityName` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '市',
  `nowCount` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '現存確診',
  `confirmedCount` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '總共確診',
  `suspectedCount` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '疑似',
  `curedCount` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '治愈',
  `deadCount` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL DEFAULT NULL COMMENT '死亡',
  PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Dynamic;

SET FOREIGN_KEY_CHECKS = 1;

 

 

 

2.是一個本地的json文件，解析，然后數據入庫。和上面其實差不多

# coding:utf8

import requests
from bs4 import BeautifulSoup
import json
import time
from pymysql import *


def mes():
    with open(r"D:\alldata.json", 'r', encoding='UTF-8') as load_f:
        load_dict = json.load(load_f)
        print(load_dict)
        load_dict = load_dict['stations']  # 拆第一層花括號
        print(len(load_dict))
        con = len(load_dict)
        print(load_dict[0]['id'])
    # data_raw = pd.DataFrame(columns=load_dict.keys())
    # data_raw = data_raw.append(load_dict, ignore_index=True)

    provinceList = []
    lenth = select()
    print("aaaaaaaaa")
    print(select())
    for item in load_dict:
        print(lenth)
        lenth = lenth + 1
        print("bbbbbbbbbbbbbbbbbbbbbb")
        print(lenth)
        id = item['id']
        type = item['type']
        name = item['name']
        dev_id = item['dev_id']
        no = item['no']
        partner = item['partner']
        enabled = item['enabled']
        agent = item['agent']
        total = item['total']
        used = item['used']
        free = item['free']
        root = item['root']
       # agent_list = item['agent_list']
       # agent_list_public = ','.join(agent_list)
       # print(agent_list_public)
        parent = item['parent']
        owner = item['owner']
        longitude = item['longitude']
        latitude = item['latitude']
        address = item['address']
        region = item['region']
        vendor = item['vendor']
        country = item['country']
        province = item['province']
        district = item['district']
        zipcoce = item['zipcoce']
        style = item['style']
        rate = item['rate']
        rate_group = item['rate_group']
        package = item['package']
        lot_rate_group = item['lot_rate_group']
        contact_phone = item['contact_phone']
        contact_name = item['contact_name']
        memo = item['memo']
        stat_total = item['stat']['total']
        stat_busy = item['stat']['busy']
        stat_idle = item['stat']['idle']
        lot_rate = item['lot_rate']
        tags = item['tags']
        price = item['price']
        #數組轉字符串
        tags_public = ','.join(tags)
        print(tags_public)
        if not lot_rate:
            print("Empty")
        else:
            print("Not Empty")
        print("========================================================================================================")
        if lot_rate==None:
            print("沒有內容")
            print(lot_rate)
            provinceList.append(
                (id, lenth, type, name, dev_id, no, partner, enabled, agent, total, used, free, root, None,
                 parent, owner, longitude, latitude, address, region, vendor, country, province, district,
                 zipcoce, style, rate, rate_group, package, lot_rate_group, contact_phone, contact_name,
                 memo, stat_total, stat_busy, stat_idle, None, None, None, None, None, None, None, None, None, None,
                 None, None, None, tags_public, price))
        else:
            print("有內容")
            print(lot_rate)
            lot_rate_id = lot_rate['id']
            lot_rate_name = lot_rate['name']
            lot_rate_agent = lot_rate['agent']
            lot_rate_unit = lot_rate['unit']
            lot_rate_free_time = lot_rate['free_time']
            lot_rate_cycle = lot_rate['cycle']
            lot_rate_max_perday = lot_rate['max_perday']
            lot_rate_max_permonth = lot_rate['max_permonth']
            lot_rate_creator = lot_rate['creator']
            lot_rate_created_time = lot_rate['created_time']
            lot_rate_update_time = lot_rate['update_time']
            lot_rate_memo = lot_rate['memo']
            #lot_rate如果有內容  該字段置為1
            provinceList.append(
            (id, lenth, type, name, dev_id, no, partner, enabled, agent, total, used, free, root, None,
             parent, owner, longitude, latitude, address, region, vendor, country, province, district,
             zipcoce, style, rate, rate_group, package, lot_rate_group, contact_phone, contact_name,
             memo, stat_total, stat_busy, stat_idle, 1, lot_rate_id, lot_rate_name, lot_rate_agent, lot_rate_unit, lot_rate_free_time,
             lot_rate_cycle, lot_rate_max_perday, lot_rate_max_permonth, lot_rate_creator,lot_rate_created_time, lot_rate_update_time,
             lot_rate_memo, tags_public, price))

        print("相數")
        print(provinceList)
    insert(provinceList)
    return NULL


def insert(provinceList):
    provinceTuple = tuple(provinceList)

    cursor = db.cursor()
    sql = "insert into park_db values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) "
    try:
        cursor.executemany(sql, provinceTuple)
        print("插入成功")
        db.commit()
    except Exception as e:
        print(e)
        db.rollback()

    cursor.close()


def select():
    sql = "select * from park_db"
    cursor = db.cursor()
    try:
        cursor.execute(sql)
        results = cursor.fetchall()
        lenth = len(results)
        db.commit()
        return lenth
    except:
        print('執行失敗，進入回調1')
        db.rollback()


def connectDB():
    try:
        db = connect(host='localhost', port=3306, user='root', password='123456', db='db3', charset='utf8')
        print("數據庫連接成功")
        return db
    except Exception as e:
        print(e)
    return NULL


if __name__ == '__main__':
    db = connectDB()
    mes()

 

 

 

 

 

 

 

 

 

 

我也忘了當時是參考了哪篇博客了，反正作者看着，聯系本人刪除即可