python爬虫同时添加用户代理和ip代理


#同时使用ip代理以及用户代理
import urllib.request
import random
#用户代理池和ip代理池
uapools = [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Maxthon/3.0)",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ;  QIHU 360EE)"]
ippools = ["61.135.217.7","118.114.77.47","111.224.104.161"]
url = 'http://www.baidu.com'
#添加用户代理以及ip代理
def ua(uapools,ippools):
    req = urllib.request.Request(url)
    req.add_header("User-Agent",random.choice(uapools))
    proxy = urllib.request.ProxyHandler({"https":random.choice(ippools)})
    opener = urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
	#将opener安装为全局
    urllib.request.install_opener(opener)
    return req
if __name__ == '__main__':
    for i in range(20):
        req = ua(uapools,ippools)
        try:
            data = urllib.request.urlopen(req)
            print(len(data.read()))
            print(data.getcode())
        except Exception as err:
            print(err)

  


免责声明!

本站转载的文章为个人学习借鉴使用,本站对版权不负任何法律责任。如果侵犯了您的隐私权益,请联系本站邮箱yoyou2525@163.com删除。



 
粤ICP备18138465号  © 2018-2025 CODEPRJ.COM