在線微博數據可視化

在線微博數據可視化，即時采集微博數據，通過不同詞雲進行展示數據

完整代碼gitee地址：https://gitee.com/lyc96/weibo

1.先來效果圖（壓壓驚）

1）輸入明星完整名字

 

2）點擊查看后，可以看到明星的言語文字可視化，有六種圖形，可以隨意切換

 

 

2.程序功能介紹

1）根據明星姓名去爬取該明星的微博言論，並存儲到文本文件（項目中不涉及任何數據庫存儲），程序中也會根據日期進行去重爬取，避免程序重復爬取同一條微博。

2）采集好的明星微博文本存儲到txt文本中，使用stylecloud進行詞雲生成圖片（有六種圖形詞雲，可以在網頁端進行切換）

 

3.python后端代碼

# -*- coding: utf-8 -*-
"""
Created on Sun Jul 19 12:03:56 2020

@author: 李運辰
"""
import requests
import time
import os
import json
from stylecloud import gen_stylecloud 
import jieba
from flask_cors import CORS
from flask import Flask,render_template,request,Response,redirect,url_for
#內網ip
app = Flask(__name__)
###此處改為自己的ip地址，在index.html中兩次也記得更改
ip="192.168.0.112"
###
root="static/data/"
pagedata="pagedata/"
textdata="textdata/"

# 睡眠時間 傳入int為休息時間，頁面加載和網速的原因 需要給網頁加載頁面元素的時間
def s(int):
    time.sleep(int)
headers = {
        
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
      }
"""初始化"""
def initialization():
    #初始化爬取記錄文本
    if not os.path.exists(root):
        os.mkdir(root)
    if not os.path.exists(root+pagedata):
        os.mkdir(root+pagedata)
    if not os.path.exists(root+textdata):
        os.mkdir(root+textdata)

def write(path,t):
    #記錄當前爬取頁數
    with open(path,"a+",encoding='utf8') as f:
        f.writelines(str(t))
        f.writelines("\n")
        
def search(name_s,url,since_id):
      
      #url = "https://m.weibo.cn/api/container/getIndex?uid=1566301073&t=0&luicode=10000011&lfid=100103type=1&q=賈玲&type=uid&value=1566301073&containerid=1076031566301073"
      start=1
      if since_id is not None and len(since_id)>1:
          url+="&since_id="+since_id
          start=0
      response = requests.get(url,headers = headers)
      
      datas = response.json()
      #print(data)
      ok = str(datas['ok'])
      try:
          with open(root+pagedata+name_s+".txt","r") as f:    #設置文件對象
              pagelist = f.read() 
      except:
          pagelist=[]
      
      if ok is not None and ok=='1':
          data = datas['data']
          since_ids = data['cardlistInfo']['since_id']
          print(since_ids)
          cards = data['cards']
          print(len(cards))
          for i in range(start,len(cards)):
               date = cards[i]['mblog']['created_at']
               if str(date) not in pagelist:
                   text1 = cards[i]['mblog']['text']
                   write(root+textdata+name_s+".txt",clean(text1))
                   write(root+pagedata+name_s+".txt",date)

"""去掉表情...,等html標簽"""
def clean(s):
    istart=-1
    try:
        istart = s.index('<')
        iend = s.index('>')
        s = s[:istart]+s[iend+1:] 
    except:
        pass    
    try:
      istart = s.index('<')
    except:
        pass
    if istart>=0:
        return clean(s)
    else:
        #print(s)
        return(s)

def geturl(name_g):
    url1="https://m.weibo.cn/api/container/getIndex?containerid=100103type=1%26q="+name_g+"&page_type=searchall"
    response = requests.get(url1,headers = headers)          
    datas = response.json()
    uid = str(datas['data']['cards'][0]['card_group'][0]['user']['id'])
    newurl = "https://m.weibo.cn/api/container/getIndex?uid="+uid+"&t=0&luicode=10000011&lfid=100103type=1&q="+name_g+"&type=uid&value="+uid+"&containerid=107603"+uid
    return newurl

def jieba_cloud(file_name,icon):
    with open(file_name,'r',encoding='utf8') as f:
        word_list = jieba.cut(f.read())
        result = " ".join(word_list) #分詞用 隔開
        #制作中文雲詞
        icon_name=""
        if icon=="1":
            icon_name=''
        elif icon=="2":
            icon_name='fas fa-dragon'
        elif icon=="3":
            icon_name='fas fa-dog'
        elif icon=="4":
            icon_name='fas fa-cat'
        elif icon=="5":
            icon_name='fas fa-dove'
        elif icon=="6":
            icon_name='fab fa-qq'
        """
        # icon_name='',#國旗
        # icon_name='fas fa-dragon',#翼龍
        icon_name='fas fa-dog',#狗
        # icon_name='fas fa-cat',#貓
        # icon_name='fas fa-dove',#鴿子
        # icon_name='fab fa-qq',#qq
        """
        picp=file_name.split('.')[0] +str(icon)+'.png'
        if icon_name is not None and len(icon_name)>0:
            gen_stylecloud(text=result,icon_name=icon_name,font_path='simsun.ttc',output_name=picp) #必須加中文字體，否則格式錯誤
        else:
            gen_stylecloud(text=result,font_path='simsun.ttc',output_name=picp) #必須加中文字體，否則格式錯誤
            
    return picp
############################flask路由
#進入首頁
@app.route('/')
def index():
    return render_template('index.html')
#獲取圖片
@app.route('/find')
def find():
    #global history
    #采集數據
    name_i = request.args.get('name')
    
    if not os.path.exists(root+textdata+name_i+'.txt'):
        u = geturl(name_i)
        search(name_i,u,"")
    #制作詞雲
    file_name = root+textdata+name_i+'.txt'
    picpath = jieba_cloud(file_name,"1")
    
    return Response(json.dumps(picpath), mimetype='application/json')
#切換圖標
@app.route('/switchs')
def switchs():
    #global history
    #采集數據
    name_i = request.args.get('name')
    icon = request.args.get('ic')
    #制作詞雲
    file_name = root+textdata+name_i+'.txt'
    picpath = jieba_cloud(file_name,str(icon))
    return Response(json.dumps(picpath), mimetype='application/json')
############################end
      
if __name__ == "__main__":    
    """初始化"""
    initialization()
    app.run(host=''+ip, port=5000,threaded=True)

 

4.完整代碼gitee地址：https://gitee.com/lyc96/weibo

 

 

關注公眾號：Python爬蟲數據分析挖掘，學習更多python知識