logstash過濾器使用ruby把時間格式轉UNIX_MS

　　日志里一般都會有時間，格式如“2020-09-04 10:08:08”，怎么轉成毫秒呢，格式如“1598609188959”？

　　假如我們的日志里僅有一個時間字段，因為我們這里轉換前是沒有毫秒的，所以可以直接轉成秒后補3個0。直接看配置：

input{
 beats {
    port => "5044"
  }
}

filter{
    mutate {
        add_field => {
            "requestTimestamp" => "%{message}"            
        }
    }

    date{
         match => ["requestTimestamp","YYYY-MM-dd HH:mm:ss"]    
         target =>"requestTimestamp"
    }
    
    ruby{
        code => "event.set('requestTimestamp',event.get('requestTimestamp').to_i*1000)"
    }
          
}
output {
  stdout { codec => rubydebug }
}

 

　　假設filebeat整不斷往logstash輸送時間日志，我們啟動后看到的將是這樣的：

{
    "requestTimestamp" => 1599211641000,
          "@timestamp" => 2020-09-04T09:27:25.754Z,
             "message" => "2020-09-04 17:27:21",
                 "log" => {
        "offset" => 483,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
            "@version" => "1",
                "host" => {
        "name" => "wulf00"
    },
               "agent" => {
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00",
            "hostname" => "wulf00",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
             "version" => "7.9.0",
                "type" => "filebeat"
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
               "input" => {
        "type" => "log"
    },
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
    "requestTimestamp" => 1599211656000,
          "@timestamp" => 2020-09-04T09:27:40.758Z,
             "message" => "2020-09-04 17:27:36",
                 "log" => {
        "offset" => 504,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
            "@version" => "1",
                "host" => {
        "name" => "wulf00"
    },
               "agent" => {
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00",
            "hostname" => "wulf00",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
             "version" => "7.9.0",
                "type" => "filebeat"
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
               "input" => {
        "type" => "log"
    },
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]

 

　　以上是時間格式為“YYYY-MM-dd HH:mm:ss”的情況，那么“YYYY-MM-dd HH:mm:ss SSS"的情況又如何呢？改下logstash配置文件：

input{
 beats {
    port => "5044"
  }
}

filter{    
    mutate {
        add_field => {
            "requestTimestamp" => "%{message}"            
        }
    }

    date{
         match => ["requestTimestamp","YYYY-MM-dd HH:mm:ss.SSS"]    
         target =>"requestTimestamp"
    }
    
    ruby{
        code => "event.set('requestTimestamp',(event.get('requestTimestamp').to_f.round(3)*1000).to_i)"
    }
          
}
output {
  stdout { codec => rubydebug }
}

 

　　重啟logstash，得到如下結果：

{
               "input" => {
        "type" => "log"
    },
          "@timestamp" => 2020-09-04T09:45:39.448Z,
                "host" => {
        "name" => "wulf00"
    },
                 "log" => {
        "offset" => 75,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
    "requestTimestamp" => 1599212733416,
             "message" => "2020-09-04 17:45:33.416",
               "agent" => {
                "type" => "filebeat",
             "version" => "7.9.0",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
            "hostname" => "wulf00",
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00"
    },
            "@version" => "1",
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
               "input" => {
        "type" => "log"
    },
          "@timestamp" => 2020-09-04T09:44:54.172Z,
                "host" => {
        "name" => "wulf00"
    },
                 "log" => {
        "offset" => 0,
          "file" => {
            "path" => "D:\\wlf\\logs\\cdr-200200006022-08971-2020090417.0.log"
        }
    },
                 "ecs" => {
        "version" => "1.5.0"
    },
    "requestTimestamp" => 1599212688414,
             "message" => "2020-09-04 17:44:48.414",
               "agent" => {
                "type" => "filebeat",
             "version" => "7.9.0",
        "ephemeral_id" => "8f29c3c9-08ea-4f6b-9508-ae187ec22f0b",
            "hostname" => "wulf00",
                  "id" => "ae375dc0-d6e2-488c-be87-2544c05b1242",
                "name" => "wulf00"
    },
            "@version" => "1",
                "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}