selenium采集頁面元素
phantomjs主要是模擬登錄
也沒多少說的，上代碼吧
from selenium import webdriver
import selenium.webdriver.support.ui as ui
import time
def crawl_cnblogs(blog_url,username,pwd):
driver = webdriver.PhantomJS()
driver.get("http://passport.cnblogs.com/user/signin?ReturnUrl=http%3A%2F%2Fwww.cnblogs.com%2F")
wait = ui.WebDriverWait(driver, 10)
wait.until(lambda dr: dr.find_element_by_id('signin').is_displayed())
driver.find_element_by_id("input1").send_keys(username)
driver.find_element_by_id("input2").send_keys(pwd)
driver.find_element_by_id("signin").click()
wait.until(lambda dr: dr.find_element_by_id('login_area').is_displayed()) #登錄成功
driver.get(blog_url)
wait.until(lambda dr: dr.find_element_by_id('mainContent').is_displayed())
time.sleep(3)
#articles = driver.find_element_by_xpath('//div[@class="postTitle"]/a') #為啥不成功
articles = driver.find_elements_by_class_name("postTitle")
for article in articles:
print article
#print article.text
#print article.text.decode("utf-8", "ignore")
# scrapy爬蟲之爬取汽車信息 編碼居然錯誤
urls = driver.find_elements_by_class_name("postTitle2")
for url in urls:
print url.get_attribute("href")
driver.save_screenshot('screen.png')
driver.quit()
if __name__ == '__main__':
crawl_cnblogs("http://www.cnblogs.com/xiaoyy3/", "xiaoyaoyou3", "------password---------")
運行結果
編碼錯誤，需要改成 print article.text.encode('gb18030')
運行結果為
