Tornado实现监控数据实时展示

本文转载自查看原文 2019-08-27 11:00 1568 Tornado相关

前言：

It has been a while since I last updated my blogs.

使用Tornado开发一个实时监控信息系统，其中包括 CUP、内存、网卡、磁盘使用率。

涉及技术

编程语言：Python

关系型数据库：MySQL

Web框架：Tornado

数据库连接驱动：mysql-connector-python

数据库ORM：sqlalchemy

服务端websocket通信：sockjs-tornado

客户端websocket通信：sockejs-client

前端：HTML、CSS、JS

图表可视化：pyecharts==0.5.11 #新版本和旧版本差异较大

获取硬件信息工具：psutil模块

pip install -i https://pypi.douban.com/simple --trusted-host pypi.douban.com -r requirements.txt
            -i:指定国内安装源                    --trusted-host:指定信任主机     -r指定依赖文件

Tornado项目目录结构设计

Monitor

-------->app01

------>models

　　 ------>static

　　　------->template

　　　------->tools

　　　------->views

manage.py

from sqlalchemy.ext.declarative import declarative_base #模型继承的父类
from sqlalchemy.dialects.mssql import BIGINT,DECIMAL,DATE,TIME,DATETIME#导入数据库字段
from sqlalchemy import Column#用于创建字段的类

Base=declarative_base()#调用
metadata=Base.metadata#创建元类

class Mem(Base):
    __tablename__='mem'
    id=Column(BIGINT,primary_key=True)
    precent=Column(DECIMAL(6,2))#小数类型 保留6位数字 2位小数
    total=Column(DECIMAL(8,2))#总量
    used=Column(DECIMAL(8,2))#使用率
    free=Column(DECIMAL(8,2))#剩余率
    create_date=Column(DATE)#创建的日期
    create_time=Column(TIME)#c创建的时间
    create_dt=Column(DATETIME)#创建的日期+时间

class Swap(Base):
    __tablename__='swap'
    id = Column(BIGINT,primary_key=True)
    precent = Column(DECIMAL(6,2))  # 小数类型 保留6位数字 2位小数
    total = Column(DECIMAL(8,2))  # 总量
    used = Column(DECIMAL(8,2))  # 使用率
    free = Column(DECIMAL(8,2))  # 剩余率
    create_date = Column(DATE)  # 创建的日期
    create_time = Column(TIME)  # c创建的时间
    create_dt = Column(DATETIME)  # 创建的日期+时间



class CPU(Base):
    __tablename__ = 'cpu'
    id = Column(BIGINT, primary_key=True)
    precent = Column(DECIMAL(6, 2))  # 小数类型 保留6位数字 2位小数
    create_date = Column(DATE)  # 创建的日期
    create_time = Column(TIME)  # c创建的时间
    create_dt = Column(DATETIME)  # 创建的日期+时间

if __name__ == '__main__':
    from sqlalchemy import create_engine
    mysql_configs={
        "db_host":'192.168.56.128',
        "db_name":"web",
        "db_port":3306,
        "db_user":"web",
        "db_pwd":"123.com"
    }
    link="mysql+mysqlconnector://{db_user}:{db_pwd}@{db_host}:{db_port}/{db_name}".format(**mysql_configs)
    engine=create_engine(link,encoding="utf-8",echo=True)
    metadata.create_all(engine)#调用元类

models.py

一、前端（SockJS）与Tornado建立web socket连接

前端使用：sockejs-client插件

//1.定义长连接
var conn = null;

//2.定义连接函数

function connect() {

    disconnect();//把之前的连接关闭掉,在创建新的连接
    //0.定义协议
    var transports = ["websocket"];
    //1.创建连接对象
    conn = new SockJS("http://" + window.location.host + "/real/time",transports);

    //2.建立连接
    conn.onopen = function () {
        console.log('连接成功')
    };
    //3.建立发送消息
    conn.onmessage = function (e) {
        console.log(e.data);
    };
    //4.建立关闭连接
    conn.onclose = function (e) {
        console.log("断开连接");
    };
    setInterval(function () {
        conn.send("system")
    },1000)
}

function disconnect() {
    if (conn != null) {
        conn.close();
        conn = null;
    }

}


if (conn == null) {
    connect();

} else {
    disconnect();
}

monitor.js

Tornado使用：sockjs

from sockjs.tornado import SockJSConnection#专门生成web_socket服务

class RealTimeHandler(SockJSConnection):
    waiters=set()#定义1个客户端连接池，所有客户端共用1个集合
    #1.建立连接（不存在重复的连接)
    def on_open(self, request):
        try:
            self.waiters.add(self)
        except Exception as e:
            print(e)
   #2.发送消息
    def on_message(self, message):#接收客户端消息
        try:
            self.broadcast(self.waiters,message)#把消息广播给所有连接的客户端
        except Exception as e:
            print(e)
    #3.关闭连接
    def on_close(self):
        try:
            self.waiters.remove(self)
        except Exception as e:
            print(e)

views_real_time.py

二、前端数据实时更新

1.CPU信息实时更新（水球图）

Tornado调用pyecharts生成前端代码（html+js+css）；

import datetime
from pyecharts import Liquid, Gauge, Pie, Line #水球图、


class Chart(object):
    def liquid_html(self,chart_id,title,val):#水球图
        liquid = Liquid(
            title="{}-{}".format(self.dt, title),
            title_pos="center",
            width="100%",
            title_color="white",
            title_text_size=14,
            height=300
        )
        liquid.chart_id=chart_id#指定 chart_id
        liquid.add("",[round(val/100,4)])#添加数据
        return liquid.render_embed()#返回html图表代码

    @property
    def dt(self):
        return datetime.datetime.now().strftime('%Y%m%dT%H:%M:%S')

chart.py

Tornado把pyecharts生成的前端代码（html+js+css），响应给浏览器渲染；

<div id="cpu_avg" style="width:100%;height:300px;"></div>
        <script type="text/javascript">

var myChart_cpu_avg = echarts.init(document.getElementById('cpu_avg'), 'light', {renderer: 'canvas'});

var option_cpu_avg = {
    "title": [
        {
            "text": "20190910T15:13:19-cpu\u5e73\u5747\u4f7f\u7528\u7387",
            "left": "center",
            "top": "auto",
            "textStyle": {
                "color": "white",
                "fontSize": 14
            },
            "subtextStyle": {
                "fontSize": 12
            }
        }
    ],
    "toolbox": {
        "show": true,
        "orient": "vertical",
        "left": "95%",
        "top": "center",
        "feature": {
            "saveAsImage": {
                "show": true,
                "title": "save as image"
            },
            "restore": {
                "show": true,
                "title": "restore"
            },
            "dataView": {
                "show": true,
                "title": "data view"
            }
        }
    },
    "series_id": 8147274,
    "tooltip": {
        "trigger": "item",
        "triggerOn": "mousemove|click",
        "axisPointer": {
            "type": "line"
        },
        "textStyle": {
            "fontSize": 14
        },
        "backgroundColor": "rgba(50,50,50,0.7)",
        "borderColor": "#333",
        "borderWidth": 0
    },
    "series": [
        {
            "type": "liquidFill",
            "data": [
                0.167
            ],
            "waveAnimation": true,
            "animationDuration": 2000,
            "animationDurationUpdate": 1000,
            "color": [
                "#294D99",
                "#156ACF",
                "#1598ED",
                "#45BDFF"
            ],
            "shape": "circle",
            "outline": {
                "show": true
            }
        }
    ],
    "legend": [
        {
            "data": [],
            "selectedMode": "multiple",
            "show": true,
            "left": "center",
            "top": "top",
            "orient": "horizontal",
            "textStyle": {
                "fontSize": 12
            }
        }
    ],
    "animation": true,
    "color": [
        "#c23531",
        "#2f4554",
        "#61a0a8",
        "#d48265",
        "#749f83",
        "#ca8622",
        "#bda29a",
        "#6e7074",
        "#546570",
        "#c4ccd3",
        "#f05b72",
        "#ef5b9c",
        "#f47920",
        "#905a3d",
        "#fab27b",
        "#2a5caa",
        "#444693",
        "#726930",
        "#b2d235",
        "#6d8346",
        "#ac6767",
        "#1d953f",
        "#6950a1",
        "#918597",
        "#f6f5ec"
    ]
};
myChart_cpu_avg.setOption(option_cpu_avg);

</script>
        </div>

tornado调用pyechart生成的前端代码

浏览器发送web socket请求给Tornado server端

Tornado不断响应浏览器pyecharts生成前端代码（html+js+css）

import json
from sockjs.tornado import SockJSConnection#专门生成web_socket服务
from app01.tools.monitor_tools import Monitor
class RealTimeHandler(SockJSConnection):
    waiters=set()#定义1个客户端连接池，所有客户端共用1个集合
    #1.建立连接（不存在重复的连接)
    def on_open(self, request):
        try:
            self.waiters.add(self)
        except Exception as e:
            print(e)
   #2.发送消息
    def on_message(self, message):#接收客户端消息，根据客户端发送过来的消息返回一些数据！
        try:
            if message =="system":
                m=Monitor()
                data={"mem":m.mem(),"swap":m.swap(),"cpu":m.cpu(),"disk":m.disk(),"net":m.net(),'dt':m.dt}
                self.broadcast(self.waiters,json.dumps(data,ensure_ascii=False))#把消息广播给所有连接的客户端
        except Exception as e:
            print(e)
    #3.关闭连接
    def on_close(self):
        try:
            self.waiters.remove(self)
        except Exception as e:
            print(e)

views_real_time.py

浏览器通过JS代码不断修改pyechart生成的前端代码

//进度条变化
function progress_status(val) {
    var data = "";
    if (val >= 0 && val < 25) {
        data = " bg-success";
    } else if (val >= 25 && val < 50) {
        data = "";
    } else if (val >= 50 && val < 75) {
        data = " bg-warning";
    } else if (val >= 75 && val <= 100) {
        data = " bg-success";
    }
    return data
}

function update_ui(e) {
    var data=JSON.parse(e.data);
    //因为pyechart 声明了变量option_cpu_avg所有我们只需要修改现有的变量option_cpu_avg即可
    option_cpu_avg.series[0].data[0]=(data['cpu']['percent_avg']/100).toFixed(4); //保留4位小数
    option_cpu_avg.title[0].text=data['dt']+'CPU平均使用率';
    //保证对option_cpu_avg的修改生效
    myChart_cpu_avg.setOption(option_cpu_avg);
    //-------------------------------------------------------
     var cpu_per = "";
    for (var k in data['cpu']['percent_per']) {
        var num = parseInt(k);
        cpu_per += "<tr><td class='text-primary' style='width: 30%'>CPU" + num + "</td>";
        cpu_per += "<td><div class='progress'><div class='progress-bar progress-bar-striped progress-bar-animated" + progress_status(data['cpu']['percent_per'][k]) + "' role='progressbar' aria-valuenow='" + data['cpu']['percent_per'][k] + "' aria-valuemin='0' aria-valuemax='100' style='width: " + data['cpu']['percent_per'][k] + "%'>" + data['cpu']['percent_per'][k] + "%</div></div></td></tr>";
    }
    document.getElementById("tb_cpu_per").innerHTML = cpu_per;

    }

//1.定义长连接
var conn = null;

//2.定义连接函数

function connect() {

    disconnect();//把之前的连接关闭掉,在创建新的连接
    //0.定义协议
    var transports = ["websocket"];
    //1.创建连接对象
    conn = new SockJS("http://" + window.location.host + "/real/time",transports);

    //2.建立连接
    conn.onopen = function () {
        console.log('连接成功')
    };
    //3.建立接收消息
    conn.onmessage = function (e) {
        // console.log(e.data);
        update_ui(e)
    };
    //4.建立关闭连接
    conn.onclose = function (e) {
        console.log("断开连接");
    };
    setInterval(function () {
        //2.1 建立连接发送消息
        conn.send("system")
    },1000)
}

function disconnect() {
    if (conn != null) {
        conn.close();
        conn = null;
    }

}


if (conn == null) {
    connect();

} else {
    disconnect();
}

monitor.js

2.内存+交互分区信息实时展示（仪表图）

import tornado.web
from app01.views.views_commen import CommnHardler
from app01.tools.monitor_tools import monitor_obj
from app01.tools.chart import Chart
#定义1个首页的视图

class IndexHandler(CommnHardler):
    def get(self,*args,**kwargs):
        cpu_info = monitor_obj.cpu()
        cpu_percent_avg_info=cpu_info ['percent_avg']#CPU平均使用率
        cpu_percent_per=cpu_info ['percent_per']     #每个CPU使用率
        mem_info=monitor_obj.mem()
        swap_info=monitor_obj.swap()
        print(swap_info)
        c=Chart()
        liquid_html=c.liquid_html(chart_id='cpu_avg',title="cpu平均使用率",val=cpu_percent_avg_info)
        self.render("index.html",**{
                                    "liquid_html":liquid_html,
                                    "cpu_percent_per_info" :cpu_percent_per,
                                    "mem_html":c.guage_html("mem","内存使用率",mem_info['percent']),
                                    "swap_html":c.guage_html('swap',"交互分区使用率",swap_info['percent']),
                                    "mem_info":mem_info,"swap_info": swap_info
                                                })

view_index.py

//进度条变化
function progress_status(val) {
    var data = "";
    if (val >= 0 && val < 25) {
        data = " bg-success";
    } else if (val >= 25 && val < 50) {
        data = "";
    } else if (val >= 50 && val < 75) {
        data = " bg-warning";
    } else if (val >= 75 && val <= 100) {
        data = " bg-success";
    }
    return data
}

function update_ui(e) {
    var data=JSON.parse(e.data);
    //因为pyechart 声明了变量option_cpu_avg所有我们只需要修改现有的变量option_cpu_avg即可
    option_cpu_avg.series[0].data[0]=(data['cpu']['percent_avg']/100).toFixed(4); //保留4位小数
    option_cpu_avg.title[0].text=data['dt']+'CPU平均使用率';
    //保证对option_cpu_avg的修改生效
    myChart_cpu_avg.setOption(option_cpu_avg);
    //-------------------------------------------------------
     var cpu_per = "";
    for (var k in data['cpu']['percent_per']) {
        var num = parseInt(k);
        cpu_per += "<tr><td class='text-primary' style='width: 30%'>CPU" + num + "</td>";
        cpu_per += "<td><div class='progress'><div class='progress-bar progress-bar-striped progress-bar-animated" + progress_status(data['cpu']['percent_per'][k]) + "' role='progressbar' aria-valuenow='" + data['cpu']['percent_per'][k] + "' aria-valuemin='0' aria-valuemax='100' style='width: " + data['cpu']['percent_per'][k] + "%'>" + data['cpu']['percent_per'][k] + "%</div></div></td></tr>";
    }
    document.getElementById("tb_cpu_per").innerHTML = cpu_per;

     /*内存实时更新*/
    option_mem.series[0].data[0].value = data['mem']['percent'];
    option_mem.title[0].text = data["dt"] + "-内存使用率";
    myChart_mem.setOption(option_mem);
    document.getElementById("mem_percent").innerText = data['mem']['percent'];
    document.getElementById("mem_total").innerText = data['mem']['total'];
    document.getElementById("mem_used").innerText = data['mem']['used'];
    document.getElementById("mem_free").innerText = data['mem']['free'];

      /*交换分区实时更新*/
    option_swap.series[0].data[0].value = data['swap']['percent'];
    option_swap.title[0].text = data["dt"] + "-交换分区使用率";
    myChart_swap.setOption(option_swap);
    document.getElementById("swap_percent").innerText = data['swap']['percent'];
    document.getElementById("swap_total").innerText = data['swap']['total'];
    document.getElementById("swap_used").innerText = data['swap']['used'];
    document.getElementById("swap_free").innerText = data['swap']['free'];

    }

//1.定义长连接
var conn = null;

//2.定义连接函数

function connect() {

    disconnect();//把之前的连接关闭掉,在创建新的连接
    //0.定义协议
    var transports = ["websocket"];
    //1.创建连接对象
    conn = new SockJS("http://" + window.location.host + "/real/time",transports);

    //2.建立连接
    conn.onopen = function () {
        console.log('连接成功')
    };
    //3.建立接收消息
    conn.onmessage = function (e) {
        // console.log(e.data);
        update_ui(e)
    };
    //4.建立关闭连接
    conn.onclose = function (e) {
        console.log("断开连接");
    };
    setInterval(function () {
        //2.1 建立连接发送消息
        conn.send("system")
    },1000)
}

function disconnect() {
    if (conn != null) {
        conn.close();
        conn = null;
    }

}


if (conn == null) {
    connect();

} else {
    disconnect();
}

monitor.js

{% extends "layout.html" %}<!--继承布局文件-->
{% block head %}
<script src="{{ static_url('echarts-liquidfill/echarts-liquidfill.min.js') }}"></script>
{% end %}
{% block content %}
<div class="row">
    <div class="col-md-12">
        <div class="card text-white bg-dark mb-3">
            <div class="card-header">CPU信息</div>
            <div class="card-body">
                <div class="row">
                    <div class="col-md-6 pad-left">
                        <table class="table table-responsive-sm table-bordered">
                            <thead>
                            <th colspan="2">所有CPU使用率</th>
                            </thead>
                            <tbody id="tb_cpu_per">
                            {% for k,v in enumerate(cpu_percent_per_info) %}
                            <tr>
                                <td class="text-primary" style="width: 30%">
                                    CPU{{ k }}
                                </td>
                                <td>
                                    <div class="progress"> <!---进度条!-->
                                        <div class="progress-bar progress-bar-striped progress-bar-animated {{ handler.progress_status(v)}}"
                                             role="progressbar" aria-valuenow="{{ v }}" aria-valuemin="0"
                                             aria-valuemax="100" style="width:{{ v }}%">{{ v }}%
                                        </div>
                                    </div>
                                </td>
                            </tr>
                            {% end %}
                            </tbody>
                        </table>
                    </div>

                    <div class="col-md-6 pad-right">
                        <div class="border border-white">{% raw liquid_html%}</div>
                    </div>
                </div>
            </div>
        </div>
    </div>
    <div class="col-md-12">
        <div class="card text-white bg-dark mb-3">
            <div class="card-header">内存/交互分区信息</div>
            <div class="card-body">
                <div class="row">
                    <div class="col-md-6 pad-left">
                        <div class="border border-white">{% raw mem_html %}</div>
                        <table class="table table-sm table-bordered">
                            <tr>
                                <td class="text-primary" style="width: 30%">使用率（%）</td>
                                 <td id="mem_percent" class="text-danger">{{ mem_info['percent'] }}</td>
                            </tr>
                            <tr>
                                <td class="text-primary" style="width: 30%">总量（GB）</td>
                                <td id="mem_total" class="text-danger">{{ mem_info['total'] }}</td>
                            </tr>

                            <tr>
                                <td class="text-primary" style="width: 30%">使用量（GB）</td>
                                 <td id="mem_used" class="text-danger">{{ mem_info['used'] }}</td>
                            <tr>
                            <tr>
                                <td class="text-primary" style="width: 30%">剩余量（GB）</td>
                                 <td id="mem_free" class="text-danger">{{ mem_info['free'] }}</td>
                            </tr>
                        </table>
                    </div>
                    <div class="col-md-6 pad-right">
                        {% raw swap_html %}
                        <table class="table table-sm table-bordered">
                            <tr>
                                <td class="text-primary" style="width: 30%">使用率（%）</td>
                                <td id="swap_percent" class="text-danger">{{swap_info['percent']}}</td>
                            </tr>
                            <tr>
                                <td class="text-primary" style="width: 30%">总量（GB）</td>
                                <td id="swap_total" class="text-danger" >{{swap_info['total']}}</td>
                            </tr>

                            <tr>
                                <td class="text-primary" style="width: 30%">使用量（GB）</td>
                                <td id="swap_used" class="text-danger">{{swap_info[ 'used']}}</td>
                            <tr>
                            <tr>
                                <td class="text-primary" style="width: 30%">剩余量（GB）</td>
                                <td id="swap_free" class="text-danger">{{swap_info['free']}}</td>
                            </tr>
                        </table>
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>
{% end %}

index.html

--------------支持web socket的协议

import json
from sockjs.tornado import SockJSConnection#专门生成web_socket服务
from app01.tools.monitor_tools import Monitor
class RealTimeHandler(SockJSConnection):
    waiters=set()#定义1个客户端连接池，所有客户端共用1个集合
    #1.建立连接（不存在重复的连接)
    def on_open(self, request):
        try:
            self.waiters.add(self)
        except Exception as e:
            print(e)
   #2.发送消息
    def on_message(self, message):#接收客户端消息，根据客户端发送过来的消息返回一些数据！
        try:
            if message =="system":
                m=Monitor()
                data={"mem":m.mem(),"swap":m.swap(),"cpu":m.cpu(),"disk":m.disk(),"net":m.net(),'dt':m.dt}
                self.broadcast(self.waiters,json.dumps(data,ensure_ascii=False))#把消息广播给所有连接的客户端
        except Exception as e:
            print(e)
    #3.关闭连接
    def on_close(self):
        try:
            self.waiters.remove(self)
        except Exception as e:
            print(e)

view_real_time.py

web socket协议

参考

pyecharts

免责声明！

本站转载的文章为个人学习借鉴使用，本站对版权不负任何法律责任。如果侵犯了您的隐私权益，请联系本站邮箱yoyou2525@163.com删除。

猜您在找 python tornado websocket 实时日志展示监控：Prometheus监控locust，grafana实时展示报告（通过boomer实现的负载生成器) 【jmeter】基于InfluxDB&Grafana的JMeter实时性能测试数据的监控和展示 Zabbix与ELK整合实现对日志数据的实时监控如何实现摄像头监控数据实时存储及传输？ flask处理数据，页面实时刷新展示 Prometheus入门到放弃(3)之Grafana展示监控数据 python爬虫等获取实时数据+Flume+Kafka+Spark Streaming+mysql+Echarts实现数据动态实时采集、分析、展示 Django实现环境实时监控功能 Flink 实现订单支付实时监控