7-13爬蟲入門之BeautifulSoup對網頁爬取內容的解析

通過beautifulsoup對json爬取的文件進行元素審查，獲取是否含有p標簽

# -*- coding:utf-8 -*-
from lxml import html
import requests
import json
import re
import scrapy
from bs4 import BeautifulSoup



#通過beautifulsoup解析文檔
def bs4analysis(html_doc):
    soup = BeautifulSoup(html_doc,"lxml")
    if soup.find_all('a'):
        print soup.a.string
        print soup.a.nextSibling
    elif html_doc.find('#')>=0:
        print '有主題'
        p=re.split('#',html_doc)
        print 'p0'+p[0]
        print 'p1'+p[1]
        print 'p2'+p[2]
    else:        
        print 'haha'
   

html_doc='''
<a class='k' href='https://m.weibo.cn/k/SHU畢業季?from=feed'>#畢業季#</a>  雲端祝福！祝全體2017屆畢業生及全球人前程似錦。 <a data-url="http://t.cn/RootR20" href="https://m.weibo.cn/p/index?containerid=230444def4f80e7a017ab35b3e37cadc001f32&url_type=39&object_type=video&pos=1&luicode=10000011&lfid=1076033243026514&featurecode=20000320&ep=F9u8aqkyn%2C3243026514%2CF9u8aqkyn%2C3243026514" data-hide=""><span class="url-icon"><img src="https://h5.sinaimg.cn/upload/2015/09/25/3/timeline_card_small_video_default.png"></span></i><span class="surl-text">秒拍視頻</a>
'''
html_doc2='''
#早安# 萬木沉酣新雨后，百昌蘇醒曉風前，四時可愛唯春色，一事能狂便少年。——王國維
'''

html_doc3='''
<a class='k' href='https://m.weibo.cn/k/通知公告?from=feed'>#通知公告#</a>南區浴室男生區因突發水管爆裂，今日起將關閉2間浴室，請各位提早安排應對。
'''
html_doc4='''
我發表了頭條文章:《共建美術學院簽約暨上海吳淞國際藝術城發展研究院揭牌儀式舉行》 <a data-url=""http://t.cn/RK2rQFs"" href=""http://media.weibo.cn/article?object_id=1022%3A2309404126988389488631&url_type=39&object_type=article&pos=1&luicode=10000011&lfid=1076033243026514&id=2309404126988389488631&ep=Fbk5FbYMp%2C3243026514%2CFbk5FbYMp%2C3243026514"" data-hide=""""><span class=""url-icon""><img src=""https://h5.sinaimg.cn/upload/2015/09/25/3/timeline_card_small_article_default.png""></span></i><span class=""surl-text"">共建美術學院簽約暨上海吳淞國際藝術城發展研究院揭牌儀式舉行</a> ​​​
'''
html_doc5='''
<a class='k' href='https://m.weibo.cn/k/SHU分享?from=feed'>#分享#</a> 斗轉星移，歲月如梭<span class=""url-icon""><img src=""//h5.sinaimg.cn/m/emoticon/icon/others/l_xin-8e9a1a0346.png"" style=""width:1em;height:1em;"" alt=""[心]""></span>
'''
if __name__ == '__main__':
    f = open('shuweibo.txt', 'r')
    fh = open('analysis.txt', 'a')
    while True:
        line = f.readline()
        if line == '':
            break
        print '*******************'
        bs4analysis(line)
        print '*******************'
        


    f.close()
    fh.close()