selenium + 抓包

攜程列表頁數據動態加載生成，接口參數加密，使用selenium + 抓包解決

初始化selenium, options參數加上，desired_capabilities參數無法使用

def dirver_init():
    options = webdriver.ChromeOptions()

    # ip = get_pro()
    # print(ip)
    # 添加代理
    # options.add_argument(('--proxy-server=http://' + ip))

    # 設置開發者模式啟動，該模式下webdriver屬性為正常值   一般反爬比較好的網址都會根據這個反爬
    options.add_experimental_option('excludeSwitches', ['enable-automation'])

    # 禁用瀏覽器彈窗
    prefs = {
        'profile.default_content_setting_values': {
            'notifications': 2
        }}
    options.add_experimental_option('prefs', prefs)

    # 添加UA
    options.add_argument(
        'user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"')

    # 將配置文件加載進webdriver
    # driver = webdriver.Chrome(options=options)
    driver = webdriver.Chrome(desired_capabilities=caps)
    return driver

動態加載，是鼠標滑動到當前窗口最下方

def dynamicLoading(driver):
    all_window_height = []  # 創建一個列表，用於記錄每一次拖動滾動條后頁面的最大高度
    all_window_height.append(driver.execute_script("return document.body.scrollHeight;"))
    while True:
        print(all_window_height)
        # driver.execute_script("document.documentElement.scrollTop = 100000")
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(3)
        heck_height = driver.execute_script("return document.body.scrollHeight;")
        if heck_height == all_window_height[-1]:  # 判斷拖動滾動條后的最大高度與上一次的最大高度的大小，相等表明到了最底部
            break
        else:
            all_window_height.append(heck_height)

判斷酒店數是否>10及是否需要動態加載，並獲取接口的相應內容

def web_selen(driver, url):
    driver.get(url)
    time.sleep(3)

    # 酒店個數
    Html = etree.HTML(driver.page_source)
    nums_content = Html.xpath("//div[contains(@class, 'filter-title')]/h3/text()")[0]
    print(nums_content)
    nums = int(re.findall(r'找到(.*)家酒店', nums_content)[0])
    print(nums)

    if nums > 10:
        # 動態加載
        dynamicLoading(driver)

        time.sleep(10)
        # 獲取接口相應內容
        request_log = driver.get_log('performance')
        for i in range(len(request_log)):
            message = json.loads(request_log[i]['message'])
            message = message['message']['params']
            # .get() 方式獲取是了避免字段不存在時報錯
            request = message.get('request')
            if (request is None):
                continue
            url = request.get('url')
            if 'HotelSearch?testab' in url:
                print(url)
                print(message['requestId'])

                try:
                    content = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': message['requestId']})
                    print(content)
                except:
                    pass
                print('-------------')

    driver.close()  # 關閉頁面
    driver.quit()