Python爬蟲 -- 喜馬拉雅爬蟲01

這是我的第一篇博客，鄙人目前正在自學Python爬蟲，對爬蟲也是了解點皮毛。下面是我做的一個小的爬蟲腳本，分享給大家，希望大家多多提出意見。

一、思路設計

　　（1）分析網頁

　　　　

　　　　在喜馬拉雅主頁找到自己想要的音頻，得到目標URL：https://www.ximalaya.com/qinggan/321787/

　　　　

　　通過分析頁面的網絡抓包，最終的到一個比較有用的json數據包

　　通過分析，得到了發送json數據包的一個有用的API接口：https://www.ximalaya.com/revision/play/album?albumId=321787&pageNum=2

　　其中album為主播的ID在頁面url中有顯示，pageNum為json數據包的“頁數”。每個json數據包有30個json數據

　　(2)設計代碼

　　　　向服務器發送請求 ---->  得到json數據包 ----> 分析json數據包 ----> 提取json數據包中的有用數據 ----> 存儲到本地MongoDB數據庫

二、代碼實例

　　　代碼共分為兩部分，執行腳本（ximalaya.py)和配置文件(config_ximalaya.py)

　　　　ximalaya.py

# -*- coding:utf-8; -*-
# Author : Bingnan Huo
# Create : 2018-12-06
import os
import time
import json
import requests

from threading import Thread
from datetime import datetime
from pymongo import MongoClient
from config_xiamalaya import *

def getWorkTimeNow(): 
    '''Acquire work time '''
    t = datetime.now()
    year = t.year
    month = t.month
    day = t.day
    hour = t.hour
    minute = t.minute
    time_str = "[%s-%s-%s-%s:%s]"%(str(year),
                       str(month),
                       str(day),
                       str(hour),
                       str(minute)
                       )
    return time_str

def getJsonData(userID,page):
    '''Get target server json data'''
    count = 0
    pa = {"albumId":userID,"pageNum":page}
    while(ERROR):
        if count > 10:
            return False
        try:
            ret = requests.get(url=INDEXURL,params=pa,headers=HEADERS,timeout=30,verify=True,proxies=None)
            ret.raise_for_status()
        except Exception as e:
            count += 1
            print(getWorkTimeNow(),end='')
            print(" [INFO] Retry...")
            continue
        else:
            ret.encoding = ret.apparent_encoding
            return ret.text

def analyseJsonData(jsonData):
    '''Analyse json data and save into MongoDB'''
    if jsonData:
        client = MongoClient()
        print(getWorkTimeNow() + " [INFO] Connected to MongoDB!")
        db = client.ximalaya# Create DataBase
        print(getWorkTimeNow() + " [INFO] Create new database!")
        table = getattr(db,TABLENAME)# Create Table
        print(getWorkTimeNow() + " [INFO] Create new table --> %s" %(TABLENAME))
        dict_obj = json.loads(jsonData)
        data = dict_obj["data"]# Json attr data
        content = data["tracksAudioPlay"]# json content
        for i in content:
            tmp_dict = {'序號':None,'名稱':None,'Url':None,'源':None,'狀態':False,'時長':None,}
            tmp_dict['序號'] = i['index']
            tmp_dict['名稱'] = i['trackName']
            tmp_dict['Url'] = "https://www.ximalaya.com" + i['trackUrl']
            tmp_dict['源'] = i['src']
            if i['isPaid']:
                tmp_dict['狀態'] = True
            tmp_dict['時長'] = i['duration']
            table.insert_one(tmp_dict)
            print(getWorkTimeNow() + " [INFO] Insert one data!")
        

def DBStart(dbpath):
    '''start MongoDB client'''
    status = os.system("start mongod --dbpath " + dbpath)
    if not status:
        print(getWorkTimeNow() + " [INFO] DataBase start!")
        return True
    else:
        print(getWorkTimeNow() + " [INFO] DataBase Failed...")
        return False
def execute(user_id,page):
    json_data = getJsonData(user_id, page)
    analyseJsonData(json_data) 

def main():
    DBStart(DBPATH)
    for page in PAGECONTIANER:
        execute(USERID, str(page))
        
  
    

if __name__ == "__main__":
    main()

　　　　config_ximalaya.py

# -*- coding:utf-8 -*-
# ximalaya.py -- config
import time

def getUnixTime():
    t = time.time()
    return str(int(t))



INDEXURL = " https://www.ximalaya.com/revision/play/album"

ERROR = True

HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0",
          
           
          }
"""
Cookie:x_xmly_traffic=utm_source%253A%2526utm_medium%253A%2526utm_campaign%253A%2526utm_content%253A%2526utm_term%253A%2526utm_from%253A; 
device_id=xm_1544076474056_jpc79kg8f1h3u6; 
Hm_lvt_4a7d8ec50cfd6af753c4f8aee3425070=1544076479; 
Hm_lpvt_4a7d8ec50cfd6af753c4f8aee3425070=1544076479
API : https://www.ximalaya.com/revision/play/album?albumId=321787&pageNum=1

"""
COOKIE = {"x_xmly_traffic":"utm_source%253A%2526utm_medium%253A%2526utm_campaign%253A%2526utm_content%253A%2526utm_term%253A%2526utm_from%253A",
          "device_id":"xm_1544076474056_jpc79kg8f1h3u6",
          "Hm_lvt_4a7d8ec50cfd6af753c4f8aee3425070":getUnixTime(),
          "Hm_lpvt_4a7d8ec50cfd6af753c4f8aee3425070":getUnixTime()
          }

DBPATH = "D:\\MongoDB\\data\\db"

TABLENAME = "Test_321787_02"

PAGECONTIANER = [i for i in range(1,10)]

USERID = "321787"

三、執行結果

　　最終的數據插入到了本地的MongoDB數據庫

　　

　　由於MongoDB為NoSQL型數據庫，該數據庫采用BOSN數據類型（json加強版）進行存儲

　　在RoboMongo中也可以用MySQL數據庫的表形式進行顯示