24-移動端app數據爬取

移動端數據爬取

安裝fiddler

真機安裝fiddler證書

修改手機代理（改成電腦ip，端口設置為fiddler的端口）

上述設置完成后我們就可以使用fiddler抓取手機端的數據了

夜神手機模擬器

移動端數據采集-案例一

找到數據接口

# -*- coding: utf-8 -*-
import requests
from lxml import etree
import json

url = "https://api.douguo.net/recipe/v2/search/0/20"
headers = {
   "User-Agent":"Mozilla/5.0 (Linux; Android 5.1.1; LIO-AN00 Build/LIO-AN00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36",
    "Cookie":"duid=65861350",
    "uuid": "feccc21d-d04b-466c-b276-98c6a7e1acef",
    "Host":"api.douguo.net",
    "language":"zh"
}
data = {
    "client": "4",
    "_session": "1599663959153866174309718910",
    "keyword": "下飯菜",
    "order": "0",
    "_vs": "400",
    "type": "0",
    "auto_play_mode": "2",
    "sign_ran": "9ce91f215449bf78a75a4a147d6bcc43",
}
response = requests.post(url=url,headers=headers,data=data).text
response2 = json.loads(response)
print(response2)

最后我只需要使用字典提取自己需要的數據就行

帶翻頁源碼

# -*- coding: utf-8 -*-
import requests
from lxml import etree
import json

#通過滑動app分析得知每頁 20遞增
#第一頁 https://api.douguo.net/recipe/v2/search/0/20
# 第二頁  https://api.douguo.net/recipe/v2/search/20/20
# 第三頁 https://api.douguo.net/recipe/v2/search/40/20
# 第四頁 https://api.douguo.net/recipe/v2/search/60/20
# 依次遞增
# 定義通用翻頁模板
url = "https://api.douguo.net/recipe/v2/search/%d"+"/20"
for pg in range(0,100,20):
    new_url = format(url%pg)
    headers = {
       "User-Agent":"Mozilla/5.0 (Linux; Android 5.1.1; LIO-AN00 Build/LIO-AN00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.136 Mobile Safari/537.36",
        "Cookie":"duid=65861350",
        "uuid": "feccc21d-d04b-466c-b276-98c6a7e1acef",
        "Host":"api.douguo.net",
        "language":"zh"
    }
    data = {
        "client": "4",
        "_session": "1599663959153866174309718910",
        "keyword": "下飯菜",
        "order": "0",
        "_vs": "400",
        "type": "0",
        "auto_play_mode": "2",
        "sign_ran": "9ce91f215449bf78a75a4a147d6bcc43",
    }
    response = requests.post(url=new_url,headers=headers,data=data).text
    response2 = json.loads(response)
    print(response2)