python爬蟲之爬取谷歌url

前言：python使用selenium庫時需要安裝chromedriver以及對應的chrome版本

代碼塊

from selenium import webdriver 
from selenium.webdriver.common.by import By #獲取元素
from selenium.webdriver.support.ui import WebDriverWait #設置獲取元素超時時間，如果獲取失敗則拋出異常
from selenium.webdriver.support import expected_conditions as EC #獲取元素
from selenium.webdriver.common.keys import Keys  #輸入鍵盤值
from selenium.common.exceptions import TimeoutException # 超時獲取元素報錯原因
import time #設置延遲
import re #正則表達式模塊
from pyquery import PyQuery as pq
browser = webdriver.Chrome()#打開google瀏覽器
urls = []
def chushi(keyword):  #初始化
    keyword = str(keyword)
    browser.get("https://www.google.com")
    wait = WebDriverWait(browser, 10) #設置獲取元素的超時時間為10s
    input = wait.until(EC.presence_of_element_located((By.NAME,"q"))) #獲取屬性name為q的元素
    input.send_keys(keyword) #輸出鍵盤值
    input.send_keys(Keys.ENTER) #輸出回車鍵
    time.sleep(3) #等待3s
    geturl()

def geturl():
    html = browser.page_source
    list1 = re.findall('<cite.*?>(.*?)<span',html) #獲取該標簽里的內容
    urls.extend(set(list1)) #set:刪除列表里的重復值 extend：在列表末尾添加內容


def next_page():
    try:
        time.sleep(3)
        wait = WebDriverWait(browser, 10)
        button = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,"#pnnext > span:nth-child(2)")))
        button.click() #模擬鼠標右鍵點擊
        time.sleep(1)
        geturl()
    except TimeoutError:
        print("請求失敗，正在重新請求")
        next_page()



def save_url():
    with open("url.txt",'w+') as f:
        url = "\n".join(urls)  #str.join：給列表里的內容之間加個換行符輸出成字符串
        f.write(url)


if __name__ == '__main__':
    page = int(input("請輸入頁數"))
    keyword = input('請輸入關鍵字')
    if page == 1:
        chushi(keyword)
        next_page()
    else:
        chushi(keyword)
        for i in range(0, page - 1):
            next_page()

    save_url()