本文我們介紹生產環境上如何通過捕捉異常recovery來完善程序設計和提高用戶體驗。
Golang異常處理
golang 的異常處理比較簡單，通常都是在程序遇到異常崩潰panic之后通過defer調用延遲函數捕捉異常，並對異常信息進行輸出和記錄。
- 異常處理代碼
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
... // 上報異常 或者 發送告警
}
}()
通過Gin中間件捕捉異常
- 內置中間件
gin在gin.Default中就使用了自帶的Recovery函數，將狀態碼置為500並輸出錯誤信息到終端
func Recovery() HandlerFunc {
return RecoveryWithWriter(DefaultErrorWriter)
}
// RecoveryWithWriter returns a middleware for a given writer that recovers from any panics and writes a 500 if there was one.
func RecoveryWithWriter(out io.Writer) HandlerFunc {
var logger *log.Logger
if out != nil {
logger = log.New(out, "\n\n\x1b[31m", log.LstdFlags)
}
return func(c *Context) {
defer func() {
if err := recover(); err != nil {
// Check for a broken connection, as it is not really a
// condition that warrants a panic stack trace.
var brokenPipe bool
if ne, ok := err.(*net.OpError); ok {
if se, ok := ne.Err.(*os.SyscallError); ok {
if strings.Contains(strings.ToLower(se.Error()), "broken pipe") || strings.Contains(strings.ToLower(se.Error()), "connection reset by peer") {
brokenPipe = true
}
}
}
if logger != nil {
stack := stack(3)
httpRequest, _ := httputil.DumpRequest(c.Request, false)
headers := strings.Split(string(httpRequest), "\r\n")
for idx, header := range headers {
current := strings.Split(header, ":")
if current[0] == "Authorization" {
headers[idx] = current[0] + ": *"
}
}
if brokenPipe {
logger.Printf("%s\n%s%s", err, string(httpRequest), reset)
} else if IsDebugging() {
logger.Printf("[Recovery] %s panic recovered:\n%s\n%s\n%s%s",
timeFormat(time.Now()), strings.Join(headers, "\r\n"), err, stack, reset)
} else {
logger.Printf("[Recovery] %s panic recovered:\n%s\n%s%s",
timeFormat(time.Now()), err, stack, reset)
}
}
// If the connection is dead, we can't write a status to it.
if brokenPipe {
c.Error(err.(error)) // nolint: errcheck
c.Abort()
} else {
c.AbortWithStatus(http.StatusInternalServerError)
}
}
}()
c.Next()
}
}
- 自定義中間件
內置的Recovery函數功能比較簡單，我們需要重新開發自定義的異常處理中間件。
在Gin-IPs項目中我們將堆棧信息上報到Redis，方便監控。同時，我們還通過traceId對該請求進行簡單的鏈路跟蹤，可以方便定位到請求日志。
// 日志打印沒必要異步處理，一般crash比較少
func Recovery() gin.HandlerFunc {
log, _ := mylog.New(
configure.GinConfigValue.ErrorLog.Path, configure.GinConfigValue.ErrorLog.Name,
configure.GinConfigValue.ErrorLog.Level, nil, configure.GinConfigValue.ErrorLog.Count)
log.Info("Test Panic")
return func(c *gin.Context) {
defer func() {
if err := recover(); err != nil {
response := route_response.Response{}
response.Data.List = []interface{}{} // 初始化為空切片，而不是空引用
traceId := c.Writer.Header().Get("X-Request-Trace-Id")
stackMsg := string(debug.Stack())
logField := map[string]interface{}{
"trace_id": traceId, // 鑒權之后可以得到唯一跟蹤ID和用戶名
"user": c.Writer.Header().Get("X-Request-User"),
"uri": c.Request.URL.Path,
"remote_addr": c.ClientIP(),
"stack": stackMsg, // 打印堆棧信息
}
c.Abort()
response.Code, response.Message = configure.ApiInnerResponseError, fmt.Sprintf("Api內部報錯，請聯系管理員(id=%s", traceId)
log.WithFields(logField).Error(err) // 輸出panic 信息
redisField := make(map[string]interface{})
for k, v := range logField {
redisField[k] = v
}
redisField["time"] = time.Now().Format("2006-01-02 15:04:05")
redisField["error"] = err
dao.ModelClient.RedisClient.HMSet(traceId, redisField) // 上報redis
c.JSON(http.StatusUnauthorized, response)
return
}
}()
c.Next()
}
}
- redis查詢異常
10.2.147.167:11700[1]> keys *
1) "445ffc1bb864000"
2) "445ff1b25864000"
10.2.147.167:11700[1]> hgetall 445ffc1bb864000
1) "time"
2) "2020-09-03 16:42:46"
3) "error"
4) "this is test panic"
5) "user"
6) "xiaoming"
7) "remote_addr"
8) "127.0.0.1"
9) "uri"
10) "/"
11) "stack"
12) "goroutine 274...." # ...省略
至此，我們將異常捕捉模塊也完成了，這其中不僅涉及到異常處理，還簡單的完成了程序內部請求鏈路跟蹤，異常信息落地到Redis也為日后的運維監控做好准備。
Github 代碼
請訪問 Gin-IPs 或者搜索 Gin-IPs