python3 爬取知乎 模擬登錄

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests
try:
    import cookielib

except:
    import http.cookiejar as cookielib
import re
import time
import os.path
try:
    from PIL import Image
except:
    pass
from bs4 import BeautifulSoup


# 構造 Request headers
#從配置表獲取
# agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3294.6 Safari/537.36'
#
# headers = {
#
#     "Host": "www.zhihu.com",
#
#     "Referer": "https://www.zhihu.com/",
#
#     'User-Agent': agent
#
# }

cookie_file = 'cookie.txt'
header={
    'Host': 'www.zhihu.com',
    'Referer': 'http://www.zhihu.com',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36',
    }
# 使用登錄cookie信息

session = requests.session()

session.cookies = cookielib.LWPCookieJar(filename=cookie_file)

try:

    session.cookies.load(ignore_discard=True)

except:

    print("Cookie 未能加載")



def get_xsrf():

    '''_xsrf 是一個動態變化的參數'''

    index_url = 'https://www.zhihu.com'

    # 獲取登錄時需要用到的_xsrf

    index_page = session.get(index_url, headers=header)

    # html = index_page.cookies

    # pattern = r'name="_xsrf" value="(.*?)"'
    #
    # # 這里的_xsrf 返回的是一個list
    #
    # _xsrf = re.findall(pattern, html)
    xsrf = index_page.request._cookies.get("_xsrf")
    return xsrf



# 獲取驗證碼

def get_captcha():

    t = str(int(time.time() * 1000))

    captcha_url = 'https://www.zhihu.com/captcha.gif?r=' + t + "&type=login"

    r = session.get(captcha_url, headers=header)

    with open('captcha.jpg', 'wb') as f:

        f.write(r.content)

        f.close()

    # 用pillow 的 Image 顯示驗證碼

    # 如果沒有安裝 pillow 到源代碼所在的目錄去找到驗證碼然后手動輸入

    try:

        im = Image.open('captcha.jpg')

        im.show()

        im.close()

    except:

        print(u'請到 %s 目錄找到captcha.jpg 手動輸入' % os.path.abspath('captcha.jpg'))

    captcha = input("please input the captcha\n>")

    return captcha



def isLogin():

    # 通過查看用戶個人信息來判斷是否已經登錄

    url = "https://www.zhihu.com/settings/profile"

    login_code = session.get(url, headers=header, allow_redirects=False).status_code

    if login_code == 200:

        return True

    else:

        return False



def login(secret, account):

    # 通過輸入的用戶名判斷是否是手機號

    if re.match(r"^1\d{10}$", account):

        print("手機號登錄 \n")

        post_url = 'https://www.zhihu.com/login/phone_num'

        postdata = {
            '_xsrf': get_xsrf(),

            'password': secret,

            'remember_me': 'true',

            'phone_num': account,

        }

    else:

        if "@" in account:

            print("郵箱登錄 \n")

        else:

            print("你的賬號輸入有問題，請重新登錄")

            return 0

        post_url = 'https://www.zhihu.com/login/email'

        postdata = {

            '_xsrf': get_xsrf(),

            'password': secret,

            'remember_me': 'true',

            'email': account,

        }

    try:

        # 不需要驗證碼直接登錄成功

        login_page = session.post(post_url, data=postdata, headers=header)

        login_code = login_page.text

        print(login_page.status_code)

        print(login_code)

    except:

        # 需要輸入驗證碼后才能登錄成功

        postdata["captcha"] = get_captcha()

        login_page = session.post(post_url, data=postdata, headers=header)

        login_code = eval(login_page.text)

        print(login_code['msg'])
    cookie_path=cookie_file
    session.cookies.save(cookie_path)
# try:
#
#     input = raw_input
#
# except:
#
#     pass



##將主頁面的用戶提問print到shell上

def  getpage(url2):

  mainpage = session.get(url2, headers=header)

  soup=BeautifulSoup(mainpage.text,'html.parser')

  tags=soup.find_all("a",class_="question_link")

  #print tags



  for tag in tags:

    print (tag.string)


def get_login_cookie(url):
    '''
    獲取保存cookie
    :param url:
    :return:
    '''
    
    if not os.path.exists(cookie_file):
        account = input('請輸入你的用戶名\n>  ')
        secret = input("請輸入你的密碼\n>  ")
        user_name = account
        passwd = secret
        login(passwd, user_name)
    try:
        cookie_jar = cookielib.LWPCookieJar(cookie_file)
        cookie_jar.load(ignore_discard=True, ignore_expires=True)
        print ('Load cookie succeeded')
    except cookielib.LoadError:
        return None
    else:
        cookie_d = {}
        for cookie in cookie_jar:
            domain = cookie.domain
            if url.find(domain) > 0:
                cookie_d[cookie.name] = cookie.value
        return cookie_d


if __name__ == '__main__':

    if isLogin():

        print('您已經登錄')

        url2='https://www.zhihu.com'

        getpage(url2)

    else:

        account = input('請輸入你的用戶名\n>  ')

        secret = input("請輸入你的密碼\n>  ")

        login(secret, account)

 

import json
import pickle
import time

from selenium import webdriver
#代碼中的chrome_options.add_argument()非常關鍵，一是要以無界面形式運行，二是禁用沙盒，否則程序報錯。
#linux上使用,打開方式以無界面打開，windows不需要這些
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 指定無界面形式運行
chrome_options.add_argument('no-sandbox') # 禁止沙盒


tianyan_cookie_file = 'tianyancha_cookie.txt'
qichacha_cookie_file = 'qichacha_cookie.txt'

#注:這些方法還需要一個插件  下面 chromedriver.exe ,也可以用火狐
# 同目錄下(腳本運行目錄)放置  chromedriver.exe  這個文件, 下載地址http://npm.taobao.org/mirrors/chromedriver/
# 部署的話最好部署在windows服務器下,安裝最新的谷歌瀏覽器,linux也可以使用,但是不能使用界面操作

#天眼查賬號登錄
def login(url):
    # dirver.maximize_window()
    dirver = webdriver.Chrome(chrome_options=chrome_options)
    dirver.get(url)
    time.sleep(3)
    # page_source = dirver.page_source
    # print(page_source)
    phoneNumber = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[2]/input')
    phone = input("你的電話")
    phoneNumber[0].send_keys(phone)

    password = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[3]/input')
    # 密碼
    passwords = input("你的密碼")
    password[0].send_keys(passwords)

    time.sleep(3)

    sublit = dirver.find_elements_by_xpath('//*[@id="web-content"]/div/div/div/div[2]/div/div[2]/div[2]/div[2]/div[5]')
    sublit[0].click()
    time.sleep(3)
    print(dirver.get_cookies())
    print(type(dirver.get_cookies()))

    with open(tianyan_cookie_file, 'wb') as handle:
        pickle.dump(dirver.get_cookies(), handle)
    time.sleep(60)
    # dirver.quit()
#使用微博模擬登陸
def login_qichacha(url):
    dirver = webdriver.Chrome(chrome_options=chrome_options)
    dirver.get(url)
    time.sleep(3)

    weibologin = dirver.find_elements_by_xpath('//*[@id="qrcodeLoginPanel"]/div[2]/div/div[3]/a[3]')
    weibologin[0].click()
    time.sleep(5)
    #用戶名  或者微博名
    phone = input("你的電話")
    dirver.find_elements_by_xpath('//*[@id="userId"]')[0].send_keys(phone)
    time.sleep(3)
    #密碼
    password = input("你的微博密碼")
    dirver.find_elements_by_xpath('//*[@id="passwd"]')[0].send_keys(password)
    time.sleep(3)
    sublimt = dirver.find_elements_by_xpath('//*[@id="outer"]/div/div[2]/form/div/div[2]/div/p/a[1]')
    sublimt[0].click()
    time.sleep(10)
    print(dirver.get_cookies())
    print(type(dirver.get_cookies()))

    with open(qichacha_cookie_file, 'wb') as handle:
        pickle.dump(dirver.get_cookies(), handle)
    dirver.close()
    # time.sleep(60)
def getCookieStr():
    #打開文件
    with open(qichacha_cookie_file, 'rb') as handle:
        b = pickle.load(handle)
        print(b)
        cookie_str = ''
        for index in b:
            cookie_str += index['name'] + '=' + index['value'] + ';'
        cookie_str = cookie_str[:-1]
        print(cookie_str)
    return cookie_str



if __name__ == '__main__':
    #天眼查
    # url = 'https://www.tianyancha.com/login'
    # login(url)

    # 企查查
    url = 'https://www.qichacha.com/user_login'
    login_qichacha(url)

    #保存cookie
    # with open(cookie_file, 'wb') as handle:
    #     pickle.dump(data, handle)
    # #打開文件
    # with open(qichacha_cookie_file, 'rb') as handle:
    #     b = pickle.load(handle)
    #     print(b)
    #     cookie_str = ''
    #     for index in b:
    #         cookie_str += index['name'] + '=' + index['value'] + ';'
    #     cookie_str = cookie_str[:-1]
    #     print(cookie_str)

 大家注意linux環境下(centos7.0下)安裝依賴參考

安裝最新版本google-chrome

yum install https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm
1
測試安裝

google-chrome-stable --headless --disable-gpu --screenshot http://www.baidu.com/
1
報錯：

[1026/154358.043744:ERROR:zygote_host_impl_linux.cc(89)] Running as
root without --no-sandbox is not supported. See
https://crbug.com/638180.

需要指定以no-sandbox方式運行。

google-chrome-stable --no-sandbox --headless --disable-gpu --screenshot http://www.baidu.com/
1
Fontconfig warning: “/etc/fonts/fonts.conf”, line 86: unknown element “blank”
[1026/154710.301159:ERROR:gpu_process_transport_factory.cc(980)] Lost UI shared context.
[1026/154711.126035:INFO:headless_shell.cc(538)] Written to file screenshot.png.

安裝chrome-driver

wget http://npm.taobao.org/mirrors/chromedriver/70.0.3538.67/chromedriver_linux64.zip
這個是解壓zip 到指定文件夾
      tar -zcvf chromedriver_linux64.zip  /usr/local/bin/
或  (二選一) 
unzip -d /usr/local/bin chromedriver_linux64.zip
1
2
python3腳本

代碼中的chrome_options.add_argument()非常關鍵，一是要以無界面形式運行，二是禁用沙盒，否則程序報錯。

# -*- coding: utf-8 -*-
import time
from selenium import webdriver

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless') # 指定無界面形式運行
chrome_options.add_argument('no-sandbox') # 禁止沙盒
driver = webdriver.Chrome(chrome_options=chrome_options)

driver.get('http://www.baidu.com/')
time.sleep(10)
print(driver.page_source) 

driver.close() 
driver.quit() 
---------------------