python selenium爬蟲工具

今天seo的同事需要一個簡單的爬蟲工具， 根據一個url地址，抓取改頁面的a連接，然后進入a連接里面的頁面再次抓取a連接

1.需要一個全局的set([])集合來保存抓取的url地址

2.由於現在單頁面也來越多，所以我們借用selenium來抓取頁面內容， 由於頁面內容比較多， 我們程序需要將滾動條滾到最下面，如：driver.execute_script("return document.body.scrollHeight;")

3.需要查找頁面的超鏈接 driver.find_elements_by_xpath("//a[@href]")

4.為了便於查看數據記錄，每抓取一個地址就記錄到日志中去（曾經嘗試過爬網完畢后再記錄，但是爬網時間太長，一旦出現異常就一條記錄都沒有了）

整個代碼如下：

from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import TimeoutException
import time
import datetime
from urllib import parse
import os

urls = set([])
def getUrl(url,host):
    driver = webdriver.Ie()
    try:
       #driver = webdriver.Firefox()
        driver.set_page_load_timeout(10)
        driver.get(url)
        #time.sleep(2)
        
        all_window_height = []
        all_window_height.append(driver.execute_script("return document.body.scrollHeight;"))
        while True:
            driver.execute_script("scroll(0,100000)")
            time.sleep(1)
            check_height = driver.execute_script("return document.body.scrollHeight;")
            if check_height == all_window_height[-1]:
                print("我已下拉完畢")
                break
            else:
                all_window_height.append(check_height) 
                print("我正在下拉")
        
        #for link in driver.find_elements_by_xpath("//*[@href]"):
        #for link in driver.find_elements_by_tag_name("a"):
        for link in driver.find_elements_by_xpath("//a[@href]"):
            try:
                tempurl1=link.get_attribute('href')
                if tempurl1.startswith("http"):
                    if tempurl1 not in urls:
                        urls.add(tempurl1)
                        log(host,url+','+tempurl1)
                        print(tempurl1)
            except:
                print(link)
    except Exception as e:
        print(e)
    finally:
        driver.quit()



def log(name,msg):
    filename='D://'+name+'.csv'
    if not os.path.exists(filename):
        with open(filename,'w') as f:
            print('create file:'+filename)
            f.write('parentUrl,currenturl'+'\n')
        f.close()
    with open(filename,'a') as f:
        f.write(msg+'\n')
    f.close()

url= input("Enter a url")
try:
    urls.clear()
    url= url.strip()
    if len(url)>0:
        host =parse.urlparse(url).netloc
        print(url+"下面的連接:")
        t1=datetime.datetime.now()
        getUrl(url,host)
        l=list(urls)
        for item in l:
            print(item+"下面的連接:")
            getUrl(item,host)
        t2=datetime.datetime.now()
        tt =(t2-t1).seconds
        minutes=tt//60
        seconds=tt%60
    print("total cost %d minutes %d seconds" % (minutes,seconds))

except Exception as e:
    print(e)

然后運行pyinstaller -F a.py 打包

關於selenium 的IE 可以參考https://blog.csdn.net/ma_jiang/article/details/96022775