引入
- 相關的門戶網站在進行登錄的時候，如果用戶連續登錄的次數超過3次或者5次的時候，就會在登錄頁中動態生成驗證碼。通過驗證碼達到分流和反爬的效果。
目錄：
- 使用超級鷹打碼平台識別驗證碼
知識點回顧
- session的創建方式
- session的作用
- proxies參數的作用
- 高匿，透明代理的區別
超級鷹打碼平台處理驗證碼的實現流程：
- 1.對攜帶驗證碼的頁面數據進行抓取 - 2.可以將頁面數據中驗證碼進行解析，驗證碼圖片下載到本地 - 3.可以將驗證碼圖片提交給三方平台進行識別，返回驗證碼圖片上的數據值 - 超級鷹打碼平台： - 1.在官網中進行注冊（普通用戶和開發者用戶） - 2.登錄普通用戶（查看余額），退出；
登錄開發者用戶： - 1.實例代碼的下載（開發文檔-》各語言SDK例子下載-》Python實例下載http://www.chaojiying.com/api-14.html）
import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:報錯題目的圖片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json() chaojiying = Chaojiying_Client('超級鷹用戶名', '超級鷹用戶名的密碼', '911742') #用戶中心>>軟件ID 生成一個替換 911742 im = open(r'D:\others\下載.jpg', 'rb').read() #本地圖片文件路徑 來替換 a.jpg 有時WIN系統須要// code = chaojiying.PostPic(im, 9004)['pic_str'].replace('|', ',') #9004 驗證碼類型 官方網站>>價格體系 print('code:',code)
- 2.創建一個軟件：軟件ID http://www.chaojiying.com/apiuser/mysoft/-》添加新的軟件 -3.使用示例代碼中的源碼文件中的代碼進行修改，讓其識別驗證碼圖片中的數據值
代碼展示：
import requests from hashlib import md5 #調用了打碼平台的相關的接口對指定的驗證碼圖片進行識別，返回圖片上的數據值 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:報錯題目的圖片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json() import requests from lxml import etree import json import time import re #1.對攜帶驗證碼的頁面數據進行抓取 url = 'https://www.douban.com/accounts/login?source=movie' headers = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Mobile Safari/537.36' } page_text = requests.get(url=url,headers=headers).text #2.可以將頁面數據中驗證碼進行解析，驗證碼圖片下載到本地 tree = etree.HTML(page_text) codeImg_url = tree.xpath('//*[@id="captcha_image"]/@src')[0] #獲取了驗證碼圖片對應的二進制數據值 code_img = requests.get(url=codeImg_url,headers=headers).content #獲取capture_id '<img id="captcha_image" src="https://www.douban.com/misc/captcha?id=AdC4WXGyiRuVJrP9q15mqIrt:en&size=s" alt="captcha" class="captcha_image">' c_id = re.findall('<img id="captcha_image".*?id=(.*?)&.*?>',page_text,re.S)[0] with open('./code.png','wb') as fp: fp.write(code_img) #獲得了驗證碼圖片上面的數據值 chaojiying = Chaojiying_Client('超級鷹用戶名', '超級鷹用戶名的密碼', '911742') #用戶中心>>軟件ID 生成一個替換 911742 im = open('code.png', 'rb').read() #本地圖片文件路徑 來替換 code.png 有時WIN系統須要// codeText = chaojiying.PostPic(im, 9101)['pic_str'] #9101 驗證碼類型 官方網站>>價格體系 #進行登錄操作 post = 'https://accounts.douban.com/login' data = { "source": "movie", "redir": "https://movie.douban.com/", "form_email": "15027900535", "form_password": "bobo@15027900535", "captcha-solution":codeText, "captcha-id":c_id, "login": "登錄", } print(c_id) login_text = requests.post(url=post,data=data,headers=headers).text with open('./login.html','w',encoding='utf-8') as fp: fp.write(login_text)
模擬登錄：爬取一些當前用戶的相關數據信息
人人網的模擬登錄（包含驗證碼處理及cookie）
import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:報錯題目的圖片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json() #人人網的模擬登錄 import requests import urllib from lxml import etree #獲取session對象 session = requests.Session() #將驗證碼圖片進行下載 headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36' } url = 'http://www.renren.com/' page_text = requests.get(url=url,headers=headers).text tree = etree.HTML(page_text) code_img_url_list = tree.xpath('//*[@id="verifyPic_login"]/@src') if len(code_img_url_list) == 0: data = { "email":"www.zhangbowudi@qq.com", "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1", "rkey":"9e75e8dc3457b14c55a74627fa64fb43", "f":"http%3A%2F%2Fwww.renren.com%2F289676607", } else: #識別驗證碼圖片中的數據值 chaojiying = Chaojiying_Client('超級鷹用戶名', '超級鷹用戶名的密碼', '911742') #用戶中心>>軟件ID 生成一個替換 911742 im = open('code.jpg', 'rb').read() #本地圖片文件路徑 來替換 code.jpg 有時WIN系統須要// code_data = chaojiying.PostPic(im, 1902)['pic_str'] #1902 驗證碼類型 官方網站>>價格體系 data = { "email":"www.zhangbowudi@qq.com", "icode":code_data, "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1", "rkey":"9e75e8dc3457b14c55a74627fa64fb43", "f":"http%3A%2F%2Fwww.renren.com%2F289676607", } code_img_url = tree.xpath('//*[@id="verifyPic_login"]/@src')[0] img_content = session.get(url=code_img_url,headers=headers).content with open('code.jpg', 'wb') as fp: fp.write(img_content) #模擬登錄 login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201914927558' #該次請求產生的cookie會被自動存儲到session對象中 session.post(url=login_url,data=data,headers=headers) url = 'http://www.renren.com/289676607/profile' main_text = session.get(url=url,headers=headers).text with open('renren.html','w',encoding='utf-8') as fp: fp.write(main_text)
import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode('utf8') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { 'user': self.username, 'pass2': self.password, 'softid': self.soft_id, } self.headers = { 'Connection': 'Keep-Alive', 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)', } def PostPic(self, im, codetype): """ im: 圖片字節 codetype: 題目類型 參考 http://www.chaojiying.com/price.html """ params = { 'codetype': codetype, } params.update(self.base_params) files = {'userfile': ('ccc.jpg', im)} r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:報錯題目的圖片ID """ params = { 'id': im_id, } params.update(self.base_params) r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers) return r.json() #模擬登錄古詩文網 s = requests.Session() headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36' } login_url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx' page_text = requests.get(url=login_url,headers=headers).text tree = etree.HTML(page_text) img_src = 'https://so.gushiwen.org'+tree.xpath('//*[@id="imgCode"]/@src')[0] img_data = s.get(url=img_src,headers=headers).content with open('./img.jpg','wb') as fp: fp.write(img_data) chaojiying = Chaojiying_Client('超級鷹用戶名', '超級鷹用戶名的密碼', '911742') #用戶中心>>軟件ID 生成一個替換 911742 im = open('img.jpg', 'rb').read() #本地圖片文件路徑 來替換 a.jpg 有時WIN系統須要// img_text = chaojiying.PostPic(im, 1902)['pic_str'] #1902 驗證碼類型 官方網站>>價格體系 #模擬登錄 url = 'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx' data = { "__VIEWSTATE":"9AsGvh3Je/0pfxId7DYRUi258ayuEG4rrQ1Z3abBgLoDSOeAUatOZOrAIxudqiOauXpR9Zq+dmKJ28+AGjXYHaCZJTTtGgrEemBWI1ed7oS7kpB7Rm/4yma/+9Q=", "__VIEWSTATEGENERATOR":"C93BE1AE", "from":"http://so.gushiwen.org/user/collect.aspx", "email":"www.zhangbowudi@qq.com", "pwd":"bobo328410948", "code":img_text, "denglu":"登錄", } page_text = s.post(url=url,headers=headers,data=data).text with open('./gushiwen.html','w',encoding='utf-8') as fp: fp.write(page_text)