前言
總算入行上班幾個月了，不得不說業務是真的不消停啊。。
本人工作上經常遇到一種場景：為甲方做自動化接口處理工具，登錄需要短信驗證碼，，
嘛算是摸索出了一套selenium代碼模板，主要解決如下痛點
- 會話超時/斷開時，又要找甲方問短信等驗證碼登錄
- 調試途中增減修改功能，算是調試中熱更新
分享一下
模板代碼
app.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import importlib
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import backend
basepath = os.path.abspath(os.path.dirname(__file__))
driver_path = os.path.join(basepath, 'chromedriver.exe')
logger = backend.logger
def init_browser(driver_path=None):
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
prefs = {
'profile.default_content_setting_values': {
'notifications': 2
}}
options.add_experimental_option('prefs', prefs)
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option("useAutomationExtension", False)
browser = webdriver.Chrome(options=options, executable_path=driver_path)
browser.maximize_window()
browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source": """
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
})
"""
})
return browser
def jump_security(wait, mouse):
wait.until(EC.presence_of_element_located((By.ID, 'details-button'))).click()
ele = wait.until(EC.presence_of_element_located((By.ID, 'proceed-link')))
mouse.move_to_element(ele).click().perform()
def init_login(driver, wait, mouse):
username_inp = wait.until(EC.presence_of_element_located((By.ID, "username")))
username_inp.send_keys("user")
password_inp = driver.find_element_by_id("password")
password_inp.send_keys("password")
class App(object):
def __new__(cls, *args, **kwargs):
if not hasattr(cls, '_instance'):
cls.error_num = 0
cls.driver = init_browser(driver_path)
cls.wait = WebDriverWait(cls.driver, 20)
cls.mouse = ActionChains(cls.driver)
cls.driver.get('https://www.target.com/login')
# jump_security(cls.wait, cls.mouse)
init_login(cls.driver, cls.wait, cls.mouse)
cls._instance = object.__new__(cls)
return cls._instance
# 模式1：client無限循環
def run_unlimited():
while True:
try:
obj = App()
input('等待登錄並進入目標頁面后，回此處按回車 >>> ')
back = backend.Backend(obj)
results = back.main()
except Exception as e:
pass
finally:
mode = input('供backend修改的阻塞暫停')
importlib.reload(backend)
# 模式2：構建本地api服務
from flask import Flask
app = Flask(__name__)
@app.route("/", methods=["GET"])
def main():
importlib.reload(backend)
back = backend.Backend(App())
results = back.main()
if __name__ == '__main__':
os.system('taskkill /im chromedriver.exe /F') # win專用，清殘留進程
os.system('taskkill /im chrome.exe /F')
run_unlimited()
# app.run()
前端有兩部分，一是單例的selenium，二是此自動化處理工具的形式：client循環形式 / api服務形式
-
單例的 _new_ 里init一些屬性，處理登錄那部分也可以放后台
-
兩種形式其實就是看形式是要主動觸發還是被動觸發，至於具體做什么就放后台
backend.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
import os
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
import simplejson
from loguru import logger
from retry import retry
from tqdm import tqdm, trange
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
basepath = os.path.abspath('./')
logger.add(f'{basepath}/logs/{os.path.basename(__file__)[:-3]}.log',
format="{level} | {time:YYYY-MM-DD HH:mm:ss} | {function}:{line} - {message}",
level="INFO", retention='5 days')
class Backend(object):
def __init__(self, obj):
self.sess = requests.session()
self.driver = obj.driver
self.sess.headers = {'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',
'Cache-Control': 'no-cache',
'Connection': 'Keep-Alive',
'Content-Length': '561',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Cookie': 'SESSION=abcdefg',
'Host': 'www.target.com',
'Referer': 'https://www.target.com/path',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
'X-Requested-With': 'XMLHttpRequest'
}
def get_cookie(self):
self.driver.find_element_by_xpath('//input[@class="e.g:trigger btn"]').click()
cookies = {_["name"]: _["value"] for _ in self.driver.get_cookies()}
return cookies
def get_headers(self):
cookies = self.get_cookie()
token = self.driver.execute_script('return window.sessionStorage.getItem("token")')
self.sess.headers.update({
'Authorization': token,
'Cookie': f'SESSION={cookies["SESSION"]}; acw_tc={cookies["acw_tc"]}'
})
@retry((json.decoder.JSONDecodeError, simplejson.errors.JSONDecodeError, KeyError, ValueError), tries=3, delay=1)
def do_api(self):
url = 'https://www.target.com/api/path'
payload = {
'params': '31b1xu0',
}
self.get_headers()
resp = self.sess.post(url, json=payload, verify=False, timeout=10)
if resp.status_code == 200:
self.pre_api_task(resp.json()) # do what you need todo
else:
raise ValueError(f'do_api failed:: {resp.text}')
def do_selenium_command(self):
self.driver.execute_script("$('p[class=imgShow]').click()")
self.driver.execute_script("document.getElementsByClassName('supportRadioOptional1 checked')[0].click();")
pagenum = int(re.search(r'共 (\d+) 頁', self.driver.page_source).group(1))
for _ in trange(pagenum, ncols=40):
self.pre_page_task() # do what you need todo
self.driver.execute_script(f"PaginationpageTable.gotoPage('next', '{_+2}', '50');")
def main(self):
self.do_selenium_command()
self.do_api()
if __name__ == '__main__':
requests.get('127.0.0.1:5000')
基於前面說的短信驗證碼，讓甲方登錄后selenium一頓操作就把api的headers補完了，可以愉快地請求接口了
需要js取參數的話可以這樣寫token = self.driver.execute_script('return window.sessionStorage.getItem("token")')
用pycharm的話可以右上角debug選擇框處編輯配置，在對應的python調試條目的配置下-> 執行處 -> 勾選使用python控制台運行，這樣程序運行后會保持內存中，仍可以ipython形式調試執行代碼
目前遇到的一些注意點：
- 渲染的頁面帶frame，需要switch_to再xpath等處理，可把
driver.page_source寫進文件判斷是否該目標頁順帶測定位 - 有時
driver.find_element_by_*無法定位，試試用js；有些JS/Jquery功能在老版IE上用不了，回用mouse處理（套娃呢喂）；連續使用js時要注意響應等待時間
- basepath處用
'./'取巧了一下(與pyinstaller打包有關)，可以基於此變量做一些本地文件處理
Last
畢竟最終是為甲方做的，程序要以甲方設備為准 即使它是win7,用pywin32定位句柄出現兼容問題即使業務網站只兼容IE內核，js部分功能無法用頭發掉光了啊
畢竟是個人摸索出的，可能有更優解，如大佬路過還請不要吝嗇交(p)流(y)一下心得