碼上快樂

相關內容    簡體    繁體

Kafka海量日志收集架構之Watcher監控告警-watcher 基礎語法與使用

本文轉載自   查看原文   2020-04-07 10:15   617    Server/ KafKa

watcher 基礎語法與使用

watcher 基礎語法與使用

  1. Xpack-Watchs基本使用：

    首先看一個watcher的例子：

    ## 創建一個watcher,比如定義一個trigger 每個10s鍾看一下input里的數據
PUT _xpack/watcher/watch/school_watcher
{
  "trigger": {
    "schedule": {
      "interval": "10s"
    }
  },
  ## 查看任務信息
  "input": {
    "search": {
      "request": {
        ## 監控具體索引
        "indices": ["school*"],
        ## body里面具體些搜索語句
        "body": {
          "size": 0,
          "query": {
            "match": {
              ## 比如索引里面name 有 hello 則進行報警
              "name": "hello"
            }
          }
        }
      }
    }
  },
  ## 對於上面的查詢結果進行比較：
  "condition": {
    ## compare進行比較
    "compare": {
      ## 上面的query查詢的結果會放入到ctx.payload中：
      ## 比如獲取 ctx.payload.hits.total  ctx.payload._shards.total 等等
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  ## transform作用：重新查詢出文檔內容賦值給ctx.payload
  "transform": {
    "search": {
      "request": {
        "indices": ["school*"],
        "body": {
          "size": 10,
          "query": {
            "match": {
              "name": "hello"
            }
          }
        }
      }
    }
  },
  ## 根據上面的查詢、比較結果，執行actions里面定義的動作（定義多種報警類型）
  "actions": {
    ## 報警名字
    "log_hello": {
      ## 防止報警風暴： 設置閾值 15m內曾經報警過, 則不報警
      "throttle_period": "15m",
      ## 報警方式：logging、mail、http等
      "logging": {
        ## 報警具體內容：使用 {{ 查詢參數 }} 進行賦值：
        "text": "Found {{ctx.payload.hits.total}} hello in the school"
      }
    }
  }
}

  2. ctx.payload取值規范：

    比如我們進行search搜索school里面name=zhangsan的數據：

    ## payload取值規范：比如我們進行search搜索school：
GET school/_search
{
  "query": {
    "match": {
      "name": "zhangsan"
    }
  }
}

    查詢結果如下：

    {
  "took": 14,
  "timed_out": false,
  "_shards": {
    "total": 2,
    "successful": 2,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 1,
    "max_score": 1.5404451,
    "hits": [
      {
        "_index": "school",
        "_type": "student",
        "_id": "1",
        "_score": 1.5404451,
        "_source": {
          "name": "zhangsan",
          "age": 25,
          "course": "elasticsearch",
          "study_date": "2018-06-15T20:30:50",
          "mark": "today is a good day"
        }
      }
    ]
  }
}

    表示查詢：ctx.payload結果集：

    ## 表示查詢：ctx.payload結果集：
{{#ctx.payload.hits.hits}} {{_source.name}} {{_source.course}} {{/ctx.payload.hits.hits}}

    比如我們進行search搜索school並采用聚合的方式來查詢terms course數據：

    GET school/_search
{
  "size": 0, 
  "aggs": {
    "myterms": {
      "terms": {
        "field": "course",
        "size": 10
      }
    }
  }
}

    查詢結果：

    {
  "took": 11,
  "timed_out": false,
  "_shards": {
    "total": 2,
    "successful": 2,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 10,
    "max_score": 0,
    "hits": []
  },
  "aggregations": {
    "myterms": {
      "doc_count_error_upper_bound": 0,
      "sum_other_doc_count": 0,
      "buckets": [
        {
          "key": "elasticsearch",
          "doc_count": 7
        },
        {
          "key": "good",
          "doc_count": 1
        },
        {
          "key": "spring",
          "doc_count": 1
        },
        {
          "key": "spring elasticsearch",
          "doc_count": 1
        }
      ]
    }
  }
}

    payload取值使用：現在想取得上面的hits.hits里面的數據內容，就可以使用如下方式：

    ## 表示查詢：ctx.payload結果集：
{{#ctx.payload.aggregations.aggsname.buckets}} {{key}} {{doc_count}} {{/ctx.payload.aggregations.aggsname.buckets}}

## 針對這里內容就是：
{{#ctx.payload.aggregations.myterms.buckets}} {{key}} {{doc_count}} {{/ctx.payload.aggregations.myterms.buckets}}

  3. watcher API使用：

    #查看一個watcher
# GET _xpack/watcher/watch/school_watcher

#刪除一個watcher
# DELETE _xpack/watcher/watch/school_watcher

#執行watcher
# POST _xpack/watcher/watch/school_watcher/_execute

#查看執行結果
GET /.watcher-history*/_search?pretty
{
  "sort" : [
    { "result.execution_time" : "desc" }
  ],
  "query": {
    "match": {
      "watch_id": "school_watcher"
    }
  }
}

  4. triggers的幾種類型

     
   #--------------------triggers的幾種類型--------------------
   #hourly、daily、weekly、monthly、yearly、cron、interval
   
   #hourly：每小時執行
   #例如：12:00, 12:15, 12:30, 12:45, 1:00, 1:15
   {
     "trigger" : {
       "schedule" : {
         "hourly" : { "minute" : [ 0, 15, 30, 45 ] }
       }
     }
   }
   
   #daily：每天執行
   #每天00:00, 12:00, and 17:00
   {
     "trigger" : {
       "schedule" : {
         "daily" : { "at" : [ "midnight", "noon", "17:00" ] }
       }
     }
   }
   #每天00:00, 00:30, 12:00, 12:30, 17:00 and 17:30
   {
     "trigger" : {
       "schedule" : {
         "daily" : {
           "at" {
             "hour" : [ 0, 12, 17 ],
             "minute" : [0, 30]
           }
         }
       }
     }
   }
   
   #weekly：指定星期幾
   #周二12:00,周五17:00
   {
     "trigger" : {
       "schedule" : {
         "weekly" : [
           { "on" : "tuesday", "at" : "noon" },
           { "on" : "friday", "at" : "17:00" }
         ]
       }
     }
   }
   #周二、周五的17：00
   {
     "trigger" : {
       "schedule" : {
         "weekly" : {
           "on" : [ "tuesday", "friday" ],
           "at" : [ "noon", "17:00" ]
         }
       }
     }
   }
   
   #monthly：指定每月哪天執行
   #每月10號中午、每月20號17:00
   {
     "trigger" : {
       "schedule" : {
         "monthly" : [
           { "on" : 10, "at" : "noon" },
           { "on" : 20, "at" : "17:00" }
         ]
       }
     }
   }
   #每月10號、20號的00:00,12:00
   {
     "trigger" : {
       "schedule" : {
         "monthly" : {
           "on" : [ 10, 20 ],
           "at" : [ "midnight", "noon" ]
         }
       }
     }
   }
   #yearly-指定月、日、時
   #每年的1月10日12:00,每年的7月20日17:00
   {
     "trigger" : {
       "schedule" : {
         "yearly" : [
           { "in" : "january", "on" : 10, "at" : "noon" },
           { "in" : "july", "on" : 20, "at" : "17:00" }
         ]
       }
     }
   }
   #每年1月10日，1月20日，12月10日，12月20日的12:00,00:00
   {
     "trigger" : {
       "schedule" : {
         "yearly" : {
           "in" : [ "jan", "dec" ],
           "on" : [ 10, 20 ],
           "at" : [ "midnight", "noon" ]
         }
       }
     }
   }
   #cron-表達式
   <seconds> <minutes> <hours> <day_of_month> <month> <day_of_week> [year]
   0 5 9 * * ?
   0 0-5 9 * * ?
   0 0/15 9 * * ?
   
   #interval-周期的
   #間隔單位：s:秒、m:分鍾、h:小時、d:天、w:星期

  5. input的幾種類型：

       #--------------------Inputs的幾種類型--------------------
   #Simple、Search、HTTP、Chain
   
   #Simple Input-靜態數據
   #每天12點觸發
   {
     "trigger" : {
       "schedule" : {
         "daily" : { "at" : "noon" }
       }
     },
     "input" : {
       "simple" : {
         "name" : "John"
       }
     },
     "actions" : {
       "reminder_email" : {
         "email" : {
           "to" : "to@host.domain",
           "subject" : "Reminder",
           "body" : "Dear {{ctx.payload.name}}, by the time you read these lines, I'll be gone"
         }
       }
     }
   }
   
   
   #Search-搜索
   {
     "input" : {
       "search" : {
         "request" : {
           "indices" : [ "logs" ],
           "body" : {
             "query" : { "match_all" : {} }
           }
         }
       }
     },
     "condition" : {
       "compare" : { "ctx.payload.hits.total" : { "gt" : 5 }}
     }
     ...
   }
   
#Http-請求
#request.host
#request.port
#request.path
#request.headers
#request.params
#request.url：request.scheme, request.host, request.port and request.params
#request.method：head、get、post、put、delete
#request.auth
#request.body
#request.proxy.host
#request.proxy.port
#request.connection_timeout
#request.read_timeout
#response_content_type：json, yaml and text
#extract
#get請求
   {
   	"input" : {
   	  "http" : {
   	    "request" : {
   	      "host" : "example.com",
   	      "port" : 9200,
   	      "path" : "/idx/_search"
   	    }
   	  }
   	}
   }
   
#含有body體內容
   {
   	"input" : {
   		"http" : {
   			"request" : {
   			  "host" : "host.domain",
   			  "port" : 9200,
   			  "path" : "/idx/_search",
   			  "body" :  "{"query" :  {  "match" : { "category" : "event"}}}"
   			}
   		}
   	}
   }
   
#含有參數的
   {
   	"input" : {
   	  "http" : {
   	    "request" : {
   	      "host" : "host.domain",
   	      "port" : "9200",
   	      "path" : "/_cluster/stats",
   	      "params" : {
   	        "human" : "true" 
   	      }
   	    }
   	  }
   	}
   }

#含有用戶密碼
   {
   	"input" : {
   	  "http" : {
   	    "request" : {
   	      "host" : "host.domain",
   	      "port" : "9200",
   	      "path" : "/myservice",
   	      "auth" : {
   	        "basic" : {
   	          "username" : "user",
   	          "password" : "pass"
   	        }
   	      }
   	    }
   	  }
   	}
   }
   
#直接請求url的
   {
   	"input" : {
   	  "http" : {
   	    "request" : {
   	      "url" : "http://api.openweathermap.org/data/2.5/weather",
   	      "params" : {
   	        "lat" : "52.374031",
   	        "lon" : "4.88969",
   	        "appid" : "<your openweathermap appid>"
   	      }
   	    }
   	  }
   	}
   }
   
#Chain-input-同時設置多個input，串行
   {
   	"input" : {
   	  "chain" : {
   	    "inputs" : [ 
   	      ## 第一步input
   	      {
   	        "first" : {
   	          "simple" : { "path" : "/_search" }
   	        }
   	      },
   	      ## 第二步input （可以去使用第一步input返回的結果）
   	      {
   	        "second" : {
   	          "http" : {
   	            "request" : {
   	              "host" : "localhost",
   	              "port" : 9200,
   	              "path" : "{{ctx.payload.first.path}}" 
   	            }
   	          }
   	        }
   	      }
   	    ]
   	  }
   	}
   }

  6. condition條件設置：如果condition條件返回true 則會觸發action 如果返回 false 則就停止，不執行action

    #--------------------條件設置--------------------
#Always Condition
"condition" : {
     "always" : {}
}
#Never Condition
   "condition" : {
     "never" : {}
}


#Compare Condition （進行和查詢的結果進行比較語法如下：）
# eq:、not_eq、gt、gte、lt、lte
## 比如錯誤條數超過了5條進行報警、響應長時間超過多少毫秒進行報警等
{
     "condition" : {
       "compare" : {
         "ctx.payload.hits.total" : { 
           "gte" : 5 
         }
     }
}

#<{expression}> 正則表達式 使用 <> 中寫正則表達式： 比如 當前時間 - 5分鍾 進行比較，如下：
{
     "condition" : {
       "compare" : {
         "ctx.execution_time" : {
           "gte" : "<{now-5m}>"
         }
     }
}


#{{path}} 比較，這個就是最開始的示例里面的獲取參數方式，如下：
{
     "condition" : {
       "compare" : {
         "ctx.payload.aggregations.status.buckets.error.doc_count" : {
           "not_eq" : "{{ctx.payload.aggregations.handled.buckets.true.doc_count}}"
         }
     }
}
   
  
#Array Compare Condition 數組比較： 比如當前的doc_count大於25 就進行報警
{
     "condition": {
       "array_compare": {
         "ctx.payload.aggregations.top_tweeters.buckets" : { 
           "path": "doc_count" ,
           "gte": { 
             "value": 25, 
           }
         }
       }
     }
}
   
#Script Condition 腳本方式
{
     "input" : {
       "search" : {
         "indices" : "log-events",
         "body" : {
           "size" : 0,
           "query" : { "match" : { "status" : "error" } }
         }
       }
     },
     "condition" : {
       "script" : {
         ## 當前返回的條數是否大於閾值，進行報警
         "inline" : "return ctx.payload.hits.total > threshold",
         "params" : {
           "threshold" : 5
         }
       }
     }
}

  7. Action 觸發器

    #--------------------Actions--------------------
#Email Action--發送郵件 

#如果使用發送郵件的報警，則需要在elasticsearch.yml中配置發送郵件服務的信息
xpack.notification.email:
   default_account: gmail_account
   account:
     gmail_account:
         profile: gmail
         smtp:
             auth: true
             starttls.enable: true
             host: smtp.gmail.com
             port: 587
             user: <username>
             password: <password>
     outlook_account:
         profile: outlook
         smtp:
             auth: true
             starttls.enable: true
             host: smtp-mail.outlook.com
             port: 587
             user: <username>
             password: <password>:
   	exchange_account:
         profile: outlook
         email_defaults:
             from: <email address of service account> 
         smtp:
             auth: true
             starttls.enable: true
             host: <your exchange server>
             port: 587
             user: <email address of service account> 
             password: <password>
   
#發送郵件
"actions" : {
     ## actions名字
     "send_email" : { 
       "email" : { 
         "to" : "'Recipient Name <recipient@example.com>'", 
         #"to" : ['Personal Name <user1@host.domain>', 'user2@host.domain'], 
         "subject" : "Watcher Notification", 
         "body" : "{{ctx.payload.hits.total}} error logs found" 
       }
     }
}

#發送含有附件信息的郵件
"actions" : {
     "email_admin" : {
       "email": {
         "to": "'John Doe <john.doe@example.com>'",
         "attachments" : {
           ## 附件方式
           "my_image.png" : { 
             "http" : { 
               "content_type" : "image.png",
               "request" : {
                 "url": "http://example.org/foo/my-image.png" 
               }
             }
           },
           ## xpack reporting插件生成方式：
           "dashboard.pdf" : {
             "reporting" : {
               "url": "http://example.org:5601/api/reporting/generate/dashboard/Error-Monitoring"
             }
           },
           ## 自定義附件
           "data.yml" : {
             "data" : {
               "format" : "yaml" 
             }
           }
         }
       }
     }
}

#Webhook Action，發送一個http請求
#發送github的issue
"actions" : {
     "create_github_issue" : {
       ## 因為發郵件到達率不是特別高，所以可以使用外部的接口調用方式
       ## 比如這里調用url為外部的手機短信接口進行發送 
       "webhook" : {
         ## 請求方式
         "method" : "POST",
         ## 外部請求地址
         "url" : "https://api.github.com/repos/<owner>/<repo>/issues",
         ## 請求報文
         "body" : "{
           "title": "Found errors in 'contact.html'",
           "body": "Found {{ctx.payload.hits.total}} errors in the last 5 minutes",
           "assignee": "web-admin",
           "labels": [ "bug", "sev2" ]
         }",
         ## 用戶名密碼
         "auth" : {
           "basic" : {
             "username" : "<username>", 
             "password" : "<password>"
           }
         }
       }
     }
}

#帶有url參數的請求
"actions" : {
     "my_webhook" : {
       "webhook" : {
         "method" : "POST",
         "host" : "mylisteningserver",
         "port" : 9200,
         "path": ":/alert",
         "params" : {
           "watch_id" : "{{ctx.watch_id}}" 
         }
       }
     }
}

#自定義header
"actions" : {
     "my_webhook" : {
       "webhook" : {
         "method" : "POST",
         "host" : "mylisteningserver",
         "port" : 9200,
         "path": ":/alert/{{ctx.watch_id}}",
         "headers" : {
           "Content-Type" : "application/yaml" 
         },
         "body" : "count: {{ctx.payload.hits.total}}"
       }
     }
}
   
#Index Action--創建索引文檔
"actions" : {
     "index_payload" : { 
       "index" : {
         "index" : "my-index", 
         "doc_type" : "my-type", 
         "doc_id": "my-id" 
       }
     }
}
   
#Logging Action--記錄日志
#level：error, warn, info, debug and trace
## 日志種類：
#category：xpack.watcher.actions.logging
"actions" : {
     "log" : { 
       "transform" : { ... }, 
       ## 日志報警
       "logging" : {
         "text" : "executed at {{ctx.execution_time}}",
         ## 日志級別
         "level": "info"
       }
     }
}

#Jira Action 與jira集成
#HipChat Action
#Slack Action
#PagerDuty Action

  8. 使用接口的形式創建一個watcher, 進行模擬：

    1. 執watcher腳本：

      ## 查詢school
GET school/student/_search
{
  "query": {
    "match_all":{}
  }
}

## 創建school_watcher
PUT _xpack/watcher/watch/school_watcher
{
  "trigger": {
    "schedule": {
      "interval": "10s"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": ["school*"],
        "body": {
          "size": 0,
          "query": {
            "match": {
              "name": "hello"
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "transform": {
    "search": {
      "request": {
        "indices": ["school*"],
        "body": {
          "size": 10,
          "query": {
            "match": {
              "name": "hello"
            }
          }
        }
      }
    }
  },
  "actions": {
    "log_hello": {
      "throttle_period": "15m",
      "logging": {
        "text": "Found {{ctx.payload.hits.total}} hello in the school"
      }
    }
  }
}

## 查看watcher執行結果
GET /.watcher-history*/_search?pretty
{
  "sort" : [
    { "result.execution_time" : "desc" }
  ],
  "query": {
    "match": {
      "watch_id": "school_watcher"
    }
  }
}

## 進行數據測試：
POST /school/student
{
 "name": "hello",
 "age": 18,
 "course": "elasticsearch",
 "study_date": "2018-08-20T20:30:50",
 "mark": "take care day day"
}

    2. 可視化操作watcher，可以啟用、禁用、添加修改、刪除watcher

  ## watch使用文章：https://www.cnblogs.com/reboot51/p/8328720.html

watcher使用：

## 創建一個watcher,比如定義一個trigger 每個10s鍾看一下input里的數據
PUT _xpack/watcher/watch/applog_error_watcher
{
  "trigger": {
    "schedule": {
      "interval": "10s"
    }
  },
  "input": {
    "search": {
      "request": {
        "indices": ["javalog-app-*"],
        "body": {
          "size": 0,
          "query": {
            "match": {
              "level": "ERROR"
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": {
        "gt": 0
      }
    }
  },
  "transform": {
    "search": {
      "request": {
        "indices": ["javalog-app-*"],
        "body": {
          "size": 10,
          "query": {
            "match": {
              "name": "hello"
            }
          }
        }
      }
    }
  },
  "actions": {
    "test_error": {
    "throttle_period": "1m",
    "webhook" : {
      "method" : "POST",
      "url" : "http://192.168.11.32:8001/watch",
      "body" : "{
        "title": "異常錯誤告警",
        "traceId": "{{#ctx.payload.hits.hits}} {{_source.traceId}} {{/ctx.payload.hits.hits}}", 
        "spanId" : "{{#ctx.payload.hits.hits}} {{_source.spanId}} {{/ctx.payload.hits.hits}}",
        "level":"告警級別P1",
        "body": "{{#ctx.payload.hits.hits}} {{_source.messageInfo}} {{/ctx.payload.hits.hits}}"
       }
      }
    }
 }
}





## {{#ctx.payload.hits.hits}} {{_source.traceId}} {{/ctx.payload.hits.hits}}
## {{#ctx.payload.hits.hits}} {{_source.spanId}} {{/ctx.payload.hits.hits}}
## {{#ctx.payload.hits.hits}} {{_source.messageInfo}} {{/ctx.payload.hits.hits}}


## 查詢error
GET javalog-app-2019.01.24/_search
{
  "query": {
    "match": {
      "level.keyword": "ERROR"
    }
  }
}


# 查看一個watcher
# GET _xpack/watcher/watch/applog_error_watcher

#刪除一個watcher
# DELETE _xpack/watcher/watch/applog_error_watcher

#執行watcher
# POST _xpack/watcher/watch/applog_error_watcher/_execute

#查看執行結果
GET /.watcher-history*/_search?pretty
{
  "sort" : [
    { "result.execution_time" : "desc" }
  ],
  "query": {
    "match": {
      "watch_id": "applog_error_watcher"
    }
  }
}


免責聲明！

本站轉載的文章為個人學習借鑒使用，本站對版權不負任何法律責任。如果侵犯了您的隱私權益，請聯系本站郵箱yoyou2525@163.com刪除。



猜您在找 watcher 基礎語法與使用 ELK實現日志監控告警 日志監控告警系統的設計與實現 zookeeper監控告警 監控告警 Zabbix監控告警 SkyWalking 監控告警 大數據學習——日志監控告警系統 第七模塊 ：微服務監控告警Prometheus架構和實踐 [運維]ELK實現日志監控告警
 
粵ICP備18138465號   © 2018-2025 CODEPRJ.COM