自學Python十一 Python爬蟲總結

　　通過幾天的學習與嘗試逐漸對python爬蟲有了一些小小的心得，我們漸漸發現他們有很多共性，總是要去獲取一系列的鏈接，讀取網頁代碼，獲取所需內容然后重復上面的工作，當自己運用的越來越熟練之后我們就會嘗試着去總結一下爬蟲的共性，試着去寫個helper類以避免重復性勞動。

　　參考:用python爬蟲抓站的一些技巧總結 zz

　　1.訪問網站 #最簡單的得到網頁代碼的方法

import urllib2
response = urllib2.urlopen("http://www.xx.com")
print response.read()

　　2.偽裝成瀏覽器(User-Agent,Referer等) #為了不被服務器禁止訪問所以還是偽裝成瀏覽器比較好

headers = {
    'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
    'Referer':'http://www.xx.com/xx',
    'Accept':'application/javascript, */*;q=0.8'
}
response = urllib2.Request(url = "http://www.xx.com",data = None,headers = headers)

　　3.Post數據轉碼

import urllib,urllib2
values = {
    'username':'xxx',
    'password':'xxx',
    'key':'xxx'
}
postdata = urllib.urlencode(values)
response = urllib2.Request(url,data = postdata)

　　4.Cookies

import urllib2,cookielib
cookie_handler = urllib2.HTTPCookieProcessor(cookielib.CookieJar())
opener = urllib2.build_opener(cookie_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)

　　5.代理服務器 #重復多次訪問同一網址 結果被封了ip或限制了訪問次數

import urllib2
proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)

　　問：如果想cookie和proxy一起用怎么辦？

　　答：urllib2.build_opener可以放多個參數，即handler 如：BaseHandler,ProxyHandler,HTTPHandler,FileHandler,FTPHandler,CacheFTPHandler等等等等

　　6.gzip #現在普遍支持gzip壓縮，我們默認獲取壓縮后的網頁，大大提高了抓取網頁的效率，減少了帶寬負荷。

import urllib2,zlib
req = urllib2.Request(url)
req.add_header('Accept-encoding', 'gzip')
response = urllib2.urlopen(req, timeout=120)
html = response.read()
gzipped = response.headers.get('Content-Encoding')
if gzipped:
    html = zlib.decompress(html, 16+zlib.MAX_WBITS)

　　7.其他

　　設置線程棧大小：棧大小顯著影響python的內存占用，方法如下：

from threading import stack_size 2 stack_size(32768*16)

　　設置超時

import socket
socket.setdefaulttimeout(10) #設置10秒后連接超時

　　失敗后重試

def get(self,req,retries=3):
    try:
        response = self.opener.open(req)
        data = response.read()
    except Exception , what:
        print what,req
        if retries>0:
            return self.get(req,retries-1)
        else:
            print 'GET Failed',req
            return ''
    return data

　　根據以上內容，我們可以寫出便於配置解決重復性工作的自己的helper類：

# -*- coding: utf-8 -*-
import cookielib, urllib, urllib2, socket
import zlib,StringIO
class HttpClient:
  __cookie = cookielib.CookieJar()
  #代理設置，需要時添加（后續設置為多代理切換）
  #__proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
  __req = urllib2.build_opener(urllib2.HTTPCookieProcessor(__cookie))#,__proxy_handler)
  __req.addheaders = [
    ('Accept', 'application/javascript, */*;q=0.8'),
    ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)')
  ]
  urllib2.install_opener(__req)

  def Get(self, url, refer=None,retries=3):
    try:
      req = urllib2.Request(url)
      req.add_header('Accept-encoding', 'gzip')
      if not (refer is None):
        req.add_header('Referer', refer)
      response = urllib2.urlopen(req, timeout=120)
      html = response.read()
      gzipped = response.headers.get('Content-Encoding')
      if gzipped:
          html = zlib.decompress(html, 16+zlib.MAX_WBITS)
      return html
    except Exception,what:
        print what
        if retries>0:
            return self.Get(url,refer,retries-1)
        else:
            print "Get Failed",url
            return ''
    #except urllib2.HTTPError, e:
    #  return e.read()
    #except socket.timeout, e:
    #  return ''
    #except socket.error, e:
    #  return ''

  def Post(self, url, data, refer=None):
    try:
      req = urllib2.Request(url, urllib.urlencode(data))
      #req = urllib2.Request(url,data)
      if not (refer is None):
        req.add_header('Referer', refer)
      return urllib2.urlopen(req, timeout=120).read()
    except urllib2.HTTPError, e:
      return e.read()
    except socket.timeout, e:
      return ''
    except socket.error, e:
      return ''

  def Download(self, url, file):
    output = open(file, 'wb')
    output.write(urllib2.urlopen(url).read())
    output.close()

  def getCookie(self, key):
    for c in self.__cookie:
      if c.name == key:
        return c.value
    return ''

  def setCookie(self, key, val, domain):
    ck = cookielib.Cookie(version=0, name=key, value=val, port=None, port_specified=False, domain=domain, domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
    self.__cookie.set_cookie(ck)

　　至於多線程就參考網上找的這段代碼好了，還支持並發。。。

from threading import Thread
from Queue import Queue
from time import sleep
#q是任務隊列
#NUM是並發線程總數
#JOBS是有多少任務
q = Queue()
NUM = 2
JOBS = 10
#具體的處理函數，負責處理單個任務
def do_somthing_using(arguments):
    print arguments
#這個是工作進程，負責不斷從隊列取數據並處理
def working():
    while True:
        arguments = q.get()
        do_somthing_using(arguments)
        sleep(1)
        q.task_done()
#fork NUM個線程等待隊列
for i in range(NUM):
    t = Thread(target=working)
    t.setDaemon(True)
    t.start()
#把JOBS排入隊列
for i in range(JOBS):
    q.put(i)
#等待所有JOBS完成
q.join()

 　　爬蟲就靠一段落吧，更深入的爬蟲框架以及html解析庫暫時放一放，讓我考慮考慮接下來的內容，是pygame還是django！

　　爬蟲demo的github地址（剛學着玩git ）：http://git.oschina.net/tabei/Python_spider