Python爬蟲三 Requests庫爬取亞馬遜報503錯誤

代碼

import requests
def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text[:10000]
    except Exception as err:
        return str(err)

if __name__ == '__main__':
    url = 'https://www.amazon.cn/dp/B077B7Z1C1?ref_=Oct_DotdV2_PC_2_GS_DOTD_56a14cb3&pf_rd_r=XF9AG6HRPAWC2MGNWJQ8&pf_rd_p=8c48638a-3752-448a-8685-5a17153fb132&pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=desktop-2'
    print(getHTMLText(url))

報503錯誤

原因

亞馬遜對請求來源進行審查
我們的Requests庫會忠實的告訴亞馬遜請求來源為‘python-requests/2.11.1’

解決辦法

import requests
def getHTMLText(url):
    try:
        kv = {'user-agent': 'Mozilla/5.0'}
        r = requests.get(url, headers=kv, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text[:3000]
    except Exception as err:
        return str(err)

if __name__ == '__main__':
    url = 'https://www.amazon.cn/dp/B077B7Z1C1?ref_=Oct_DotdV2_PC_2_GS_DOTD_56a14cb3&pf_rd_r=XF9AG6HRPAWC2MGNWJQ8&pf_rd_p=8c48638a-3752-448a-8685-5a17153fb132&pf_rd_m=A1AJ19PSB66TGU&pf_rd_s=desktop-2'
    print(getHTMLText(url))