Python爬蟲——抓取貼吧帖子

抓取百度貼吧帖子

按照這個學習教程，一步一步寫出來，中間遇到很多的問題，一一列舉

首先， 獲得 標題 和 貼子總數

# -*- coding:utf-8 -*-
#!/user/bin/python


import urllib
import urllib2
import re


class BDTB:
    #初始化，傳入基地址，是否只看樓主的參數
    def __init__(self, baseUrl, seeLZ):
        self.baseURL = baseUrl
        self.seeLZ = '?see_lz=' + str(seeLZ)

    #傳入頁碼，獲取該頁帖子的代碼
    def getPage(self, pageNum):
        try:
            url = self.baseURL + self.seeLZ + '&pn=' + str(pageNum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            return response.read()
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print u"連接百度貼吧失敗,錯誤原因", e.reason
                return None

    def getTitle(self):
        page = self.getPage(1)
        pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>', re.S)
        result = re.search(pattern, page)
        if result:
            print result.group(1)
            return result.group(1).strip()
        else:
            return None

    #得到帖子頁數
    def getPageNum(self):
        page = self.getPage(1)
        pattern = re.compile('<li class="l_reply_num.*?<span.*?>(.*?)</span',re.S)
        result = re.search(pattern, page)
        if result:
            print "回復個數："
            print result.group(1)
            return result.group(1).strip()
        else:
            return None


baseURL = 'http://tieba.baidu.com/p/3138733512'
bdtb = BDTB(baseURL, 1)
bdtb.getTitle()
bdtb.getPageNum()

PS：我用的火狐瀏覽器，查看網頁源代碼，鼠標右擊查看 獲得 快捷鍵 Ctrl-U

接下來 抓取 樓層的內容，寫好的 程序如下

import urllib
import urllib2
import re


class BDTB:
    #初始化，傳入基地址，是否只看樓主的參數
    def __init__(self, baseUrl, seeLZ):
        self.baseURL = baseUrl
        self.seeLZ = '?see_lz=' + str(seeLZ)

    #傳入頁碼，獲取該頁帖子的代碼
    def getPage(self, pageNum):
        try:
            url = self.baseURL + self.seeLZ + '&pn=' + str(pageNum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            return response.read()
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print u"連接百度貼吧失敗,錯誤原因", e.reason
                return None

    def getTitle(self):
        page = self.getPage(1)
        pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>', re.S)
        result = re.search(pattern, page)
        if result:
            print result.group(1)
            return result.group(1).strip()
        else:
            return None

    #得到帖子頁數
    def getPageNum(self):
        page = self.getPage(1)
        pattern = re.compile('<li class="l_reply_num.*?<span.*?>(.*?)</span',re.S)
        result = re.search(pattern, page)
        if result:
            print "回復個數："
            print result.group(1)
            return result.group(1).strip()
        else:
            return None

    def getContent(self,page):
        pattern = re.compile('<div id="post_content_.*? class="d_post_content j_d_post_content ">(.*?)</div>',re.S)
        items = re.findall(pattern,page)
        for item in items:
            print item

baseURL = 'http://tieba.baidu.com/p/3138733512'
bdtb = BDTB(baseURL, 1)
bdtb.getTitle()
bdtb.getPageNum()
bdtb.getContent(1)

但是運行之后一直報錯，如下圖：

 

 檢查代碼無數次后，終於.....發現 getContent中 沒有獲取頁碼 T_T  在這個函數首句加上 

page = self.getPage(1)  

即可！！！

終於得到了內容部分，用一下工具類 可將亂七八糟的圖片什么的代碼去掉

#處理頁面標簽類
class Tool:
    #去除img標簽,7位長空格
    removeImg = re.compile('<img.*?>| {7}|')
    #刪除超鏈接標簽
    removeAddr = re.compile('<a.*?>|</a>')
    #把換行的標簽換為\n
    replaceLine = re.compile('<tr>|<div>|</div>|</p>')
    #將表格制表<td>替換為\t
    replaceTD= re.compile('<td>')
    #把段落開頭換為\n加空兩格
    replacePara = re.compile('<p.*?>')
    #將換行符或雙換行符替換為\n
    replaceBR = re.compile('<br><br>|<br>')
    #將其余標簽剔除
    removeExtraTag = re.compile('<.*?>')
    def replace(self,x):
        x = re.sub(self.removeImg,"",x)
        x = re.sub(self.removeAddr,"",x)
        x = re.sub(self.replaceLine,"\n",x)
        x = re.sub(self.replaceTD,"\t",x)
        x = re.sub(self.replacePara,"\n    ",x)
        x = re.sub(self.replaceBR,"\n",x)
        x = re.sub(self.removeExtraTag,"",x)
        #strip()將前后多余內容刪除
        return x.strip()

 

最后最后，就是這樣的了..

# -*- coding:utf-8 -*-
#!/user/bin/python

import urllib
import urllib2
import re



#處理頁面標簽類
class Tool:
    #去除img標簽,7位長空格
    removeImg = re.compile('<img.*?>| {7}|')
    #刪除超鏈接標簽
    removeAddr = re.compile('<a.*?>|</a>')
    #把換行的標簽換為\n
    replaceLine = re.compile('<tr>|<div>|</div>|</p>')
    #將表格制表<td>替換為\t
    replaceTD= re.compile('<td>')
    #把段落開頭換為\n加空兩格
    replacePara = re.compile('<p.*?>')
    #將換行符或雙換行符替換為\n
    replaceBR = re.compile('<br><br>|<br>')
    #將其余標簽剔除
    removeExtraTag = re.compile('<.*?>')
    def replace(self,x):
        x = re.sub(self.removeImg,"",x)
        x = re.sub(self.removeAddr,"",x)
        x = re.sub(self.replaceLine,"\n",x)
        x = re.sub(self.replaceTD,"\t",x)
        x = re.sub(self.replacePara,"\n    ",x)
        x = re.sub(self.replaceBR,"\n",x)
        x = re.sub(self.removeExtraTag,"",x)
        #strip()將前后多余內容刪除
        return x.strip()

    
class BDTB:
    #初始化，傳入基地址，是否只看樓主的參數
    def __init__(self, baseUrl, seeLZ, floorTag):
        self.baseURL = baseUrl
        self.seeLZ = '?see_lz=' + str(seeLZ)
        self.tool = Tool()
        #全局file變量，文件寫入操作對象
        self.file = None
        #樓層標號， 初始化為1
        self.floor = 1
        #默認標題
        self.defaultTitle = u"百度某某貼吧"
        #是否寫入樓層分隔符標記
        self.floorTag = floorTag

    #傳入頁碼，獲取該頁帖子的代碼
    def getPage(self, pageNum):
        try:
            url = self.baseURL + self.seeLZ + '&pn=' + str(pageNum)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            return response.read().decode('utf-8')
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                print u"連接百度貼吧失敗,錯誤原因", e.reason
                return None

    #獲得帖子標題
    def getTitle(self,page):
        page = self.getPage(1)
        pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>', re.S)
        result = re.search(pattern, page)
        if result:
            #print result.group(1)
            return result.group(1).strip()
        else:
            return None

    #得到帖子頁數
    def getPageNum(self,page):
        page = self.getPage(1)
        pattern = re.compile('<li class="l_reply_num.*?<span.*?>(.*?)</span',re.S)
        result = re.search(pattern, page)
        if result:
            #print "回復個數："
            #print result.group(1)
            return result.group(1).strip()
        else:
            return None

    #獲得帖子的內容
    def getContent(self,page):
        page = self.getPage(1)
        pattern = re.compile('<div id="post_content_.*?>(.*?)</div>',re.S)
        items = re.findall(pattern,page)
        contents = []
        floor = 1
        for item in items:
            content = "\n" + self.tool.replace(item) + "\n"
            contents.append(content.encode('utf-8'))
            #print self.tool.replace(item)
            #floor += 1
        return contents
    
    def setFileTitle(self,title):
        if title is not None:
            self.file = open(title + ".txt", "w+")
        else:
            self.file = open(self.defaultTitle + ".txt", "w+")

    def writeData(self,contents):
        for item in contents:
            if self.floorTag == '1':
                floorline = "\n" + str(self.floor) + u"-------------------------------------\n"
                self.file.write(floorline)
            self.file.write(item)
            self.floor += 1

    def start(self):
        indexPage = self.getPage(1)
        pageNum = self.getPageNum(indexPage)
        title = self.getTitle(indexPage)
        self.setFileTitle(title)
        if pageNum == None:
            print "URL已失效，請重試"
            return
        try:
            print "該帖子共有" + str(pageNum) + "頁"
            for i in range(1,int(pageNum) + 1):
                print "正在寫入第" + str(i) + "頁數據"
                page = self.getPage(i)
                contents = self.getContent(page)
                self.writeData(contents)
        except IOError,e:
            print "寫入異常，原因" + e.message
        finally:
            print "Succeed~"
                               

print u"請輸入帖子代碼"
baseURL = 'http://tieba.baidu.com/p/' + str(raw_input(u'http://tieba.baidu.com/p/'))
seeLZ = raw_input("是否只看樓主，是輸入1，否輸入0\\n")
floorTag = raw_input("是否寫入樓層信息，是輸入1，否輸入0\\n")
bdtb = BDTB(baseURL, seeLZ,floorTag)
bdtb.start()

關於decode和encode知識，查看 這個

關於raw_input, 查看 這個