python+selenium操作chrome瀏覽器抓取網頁解決方案

以下操作均是在ubuntu系統下運行

from selenium import webdriver
from scrapy.selector import Selector

#操作chrome瀏覽器抓取淘寶

driver = webdriver.Chrome()
driver.get('淘寶鏈接')
print(driver.page_source)
t_selector = Selector(text=driver.page_source)
tm_price = t_selector.xpath('//*[@id="J_StrPriceModBox"]/dd/span/text()').extract_first()
print(tm_price)
driver.quit()

#操作谷歌瀏覽器登錄知戶

browser = webdriver.Chrome()
browser.get("https://www.zhihu.com/signin?next=%2Fexplore")
browser.find_element_by_xpath('//div[@class="SignFlow-accountInput Input-wrapper"]/input[@name="username"]').send_keys("18262031725")
browser.find_element_by_xpath('//div[@class="Input-wrapper"]/input[@name="password"]').send_keys("....")
browser.find_element_by_xpath('//div[@class="Login-options"]/button[@type="submit"]').click()

#操作瀏覽器登錄微薄，並且下拉

browser = webdriver.Chrome()
browser.get("https://weibo.com/")
browser.implicitly_wait(10)
browser.find_element_by_xpath('//input[@id="loginname"]').send_keys('18262031725')
browser.find_element_by_xpath('//div[@class="input_wrap"]/input[@type="password"]').send_keys('。。。。')
browser.find_element_by_xpath('//div[@class="info_list login_btn"]/a').click()
import time
time.sleep(5)
browser.execute_script("window.scrollTo(0,document.body.scrollHeight);var lenofPage=document.body.scrollHeight;return lenofPage")

#設置不加載圖片

chrome_opt = webdriver.ChromeOptions()
prefs = {"profile.managed_default_content_settings.images":2} # 不加載圖片
chrome_opt.add_experimental_option("prefs",prefs)
browser = webdriver.Chrome(chrome_options=chrome_opt)
browser.get("https://weibo.com/")

#phantomjs,無界面瀏覽器，多進程情況下phantomjs性能下降非常嚴重,不推薦用

driver = webdriver.PhantomJS()
driver.get('https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.1.1bb93598Z5K5Sz&id=45696640046&skuId=3361138965307&user_id=728443962&cat_id=2&is_b=1&rn=7fd2566c963598402926dd52a6776c8c')
print(driver.page_source)
t_selector = Selector(text=driver.page_source)
tm_price = t_selector.xpath('//*[@id="J_StrPriceModBox"]/dd/span/text()').extract_first()
print(tm_price)
driver.quit()

#通過PyVirtualDisplay可以實現無界面,需要安裝xvfb

sudo apt-get install xvfb
pip install xvfbwrapper

from pyvirtualdisplay import Display
display = Display(visible=0,size=(800,600))
display.start()
browser = webdriver.Chrome()
browser.get('https://weibo.com/')
print(browser.page_source)

#推薦Headless Chrome指在headless模式下運行谷歌瀏覽器，實現無界面

opt = webdriver.ChromeOptions()
opt.set_headless()
opt.add_argument('--disable-gpu')
browser = webdriver.Chrome(options=opt)
browser.get("https://weibo.com/")
print(browser.page_source)
browser.close()