requests的封装(user-agent,proxies)

import requests 
# 1. headers

url = 'https://api.github.com/some/endpoint' headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36'} r = requests.get(url, headers=headers)

#  2. proxies

proxies = { 'http': 'http://10.10.1.10:3128', 'https': 'http://10.10.1.10:1080', } requests.get('http://example.org', proxies=proxies)


@example version_1.0 封装 requests

def proxy_request(url):
   def get_proxy(retries=5):
      if retries:
         proxy = requests.get('http://localhost:5000/proxy/get')
         if proxy.status_code == 200:
            return proxy.text
         else:
            get_proxy(retries - 1)
      else:
         return None

   proxy = get_proxy()
   if proxy is not None:
      proxies = {"http": "http://%s" % proxy}
   else:
      print('***********************代理池没有IP可用啦!!!**********************')
      time.sleep(15)
      proxies = {"http": "http://%s" % "127.0.0.0:1080"}

   headers = {'user-agent':UserAgent().random}
   r = requests.get(url=url,headers=headers,proxies=proxies)
   return r