定時獲取阿里雲MySQL慢查詢日志

項目背景：每天早上10點30分定時發送釘釘信息
軟件版本：Centos 7.4， python 3.6，docker 19.03.12， docker-compose 1.25.5

創建 python 項目

安裝依賴

slow_log.py

from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.acs_exception.exceptions import ClientException
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkrds.request.v20140815.DescribeSlowLogsRequest import DescribeSlowLogsRequest
import json
import openpyxl
from datetime import datetime, date, timedelta
import dingtalk

# 全局變量，保存返回符合要求的慢查詢記錄
total_record = []

# 全局變量，設置查詢的日期
yesterday = (date.today() + timedelta(days=-1)).strftime("%Y-%m-%d")
dbtime = yesterday + 'Z'

# 阿里雲 SDK 配置信息, 配置RAM，給予阿里雲RDS只讀權限
client = AcsClient('xxxxxxxxxxxx', 'xxxxxxxxxx', 'cn-shenzhen')

request = DescribeSlowLogsRequest()
request.set_accept_format('json')

# 慢查詢日志統計的開始時間和結束時間，這里我們獲取的前一天的數據
request.set_StartTime(dbtime)
request.set_EndTime(dbtime)


# 獲取返回慢查詢的記錄數
def get_total_count(dbinstance):
    request.set_DBInstanceId(dbinstance)
    response = client.do_action_with_exception(request)
    result_json = json.loads(str(response, encoding='utf-8'))

    return result_json['TotalRecordCount']


# 獲取慢查詢記錄，增加到全局變量：total_record 中
def get_record(dbinstance, pagenumber=1):
    request.set_DBInstanceId(dbinstance)
    request.set_PageNumber(pagenumber)
    response = client.do_action_with_exception(request)
    result_json = json.loads(str(response, encoding='utf-8'))

    for p in range(result_json['PageRecordCount']):
        # 判斷的規則是 同一條SQL 的慢查詢記錄一天內執行超過 100次，且平均時間大於 1s
        if result_json['Items']['SQLSlowLog'][p]['MySQLTotalExecutionCounts'] > 100 and (result_json['Items']['SQLSlowLog'][p]['MySQLTotalExecutionTimes'] / result_json['Items']['SQLSlowLog'][p]['MySQLTotalExecutionCounts'] > 1):
            total_record.append(result_json['Items']['SQLSlowLog'][p])


# 生成 xlsx 文件
def generate_xlsx(total_record, dbinstance):
    if total_record:
        filename = "/export_xlsx/" + dbinstance + "_mysql_slow_sql_" + yesterday + ".xlsx"
        workbook = openpyxl.Workbook()
        sheet = workbook.active

        # 添加列標題
        data1 = list(total_record[0].keys())
        sheet.append(data1)

        # 添加值
        for i in range(len(total_record)):
            data2 = list(total_record[i].values())
            sheet.append(data2)
        
        # 保存 xlsx
        workbook.save(filename=filename)

        # 發送釘釘消息
        msg = "http://192.168.0.200:81" + filename
        dingtalk.send_msg(msg)


def main(dbinstance):
    # 每次調用 main 函數前，先清空 total_record 列表
    total_record.clear()
    # 獲取總的慢查詢記錄數
    total_record_count = get_total_count(dbinstance)
    # 獲取慢查詢頁碼，阿里雲api默認返回一頁30條記錄
    total_record_page = total_record_count // 30 + 1
    # 循環所有頁碼
    for p in range(1, total_record_page+1):
        get_record(dbinstance, p)

    # 生成 xlsx
    generate_xlsx(total_record, dbinstance)


if __name__ == '__main__':
    # 對兩個 RDS實例 進行檢測
    dbinstance = "rr-xxxxxxxxxxxxxxxxx"
    main(dbinstance)

    dbinstance = "rm-xxxxxxxxxxxxxxxxx"
    main(dbinstance)

dingtalk.py

import json
import requests


# 傳入 msg 參數，發送釘釘消息
def send_msg(msg):
    url = 'https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' #  釘釘群機器人url
    parameter = {
        "msgtype": "text",
        "text": {
            "content": "雲服務 -- 數據庫慢查詢日志已生成，下載請點擊：%s ，請及時查看(該鏈接只可以在公司內網打開)~" % msg   # 這里我定義的關鍵字是 雲服務，記得在 釘釘群機器人的安全設置上加上
        },
    }
    headers = {
        'Content-Type': 'application/json'
    }
    requests.post(url, data=json.dumps(parameter), headers=headers)

在 PyCharm 中 使用命令導出依賴

pip freeze > requirements.txt

把程序打包進 docker

創建一個基礎鏡像

mkdir /data/demo -p && cd /data/demo

vi Dockerfile
FROM python:3.6
WORKDIR /data

COPY requirements.txt ./
RUN pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple/

# 把 requirements.txt 上傳到 /data/demo 上

docker build -t slow_sql_template:0.1 ./

創建 nginx 鏡像用來提供MySQL慢日志下載服務

mkdir /data/nginx4slow_sql -p && cd /data/nginx4slow_sql

vi docker-compose.yml
version: '3.4'
services:
  nginx4slow_sql: 
    image: nginx:1.19
    ports:
      - 81:80           # 使用81端口對外提供服務
    volumes:
      - /data/export_xlsx:/usr/share/nginx/html/export_xlsx  # 把宿主機的 /data/export_xlsx 映射到容器中，提供文件下載服務

mkdir -p /data/export_xlsx

docker-compose up -d 

創建應用鏡像

mkdir /data/alarm4slow_sql -p && cd /data/alarm4slow_sql

vi  Dockerfile 
FROM slow_sql_template:0.1
WORKDIR /data
COPY . .
RUN mkdir /export_xlsx
 
# 把程序 slow_log.py 和 dingtalk.py 上傳到該目錄

docker build -t alarm4slow_log:0.1 ./

vi docker-compose.yml
version: '3.4'
services:
  alarm4slow_sql: 
    image: alarm4slow_log:0.1
    command: python slow_log.py 
    volumes:
      - /data/export_xlsx:/export_xlsx   # 把宿主機的 /data/export_xlsx 映射到容器中的 /export_xlsx 目錄中

docker-compose up -d 

運行結果：

添加 Linux 定時任務觸發

crontab -e
# start alarm4slow_sql  
30 10 * * * cd /data/alarm4slow_sql && /usr/local/bin/docker-compose up -d > /dev/null 2>&1