在Logstash的配置文件中對日志事件進行區分

1、多個日志文件作為輸入源

input {
    # 通過給日志事件定義類型來區分
    file {
        path => ["/var/log/nginx/access.log"]
        type => "nginx_access"
        start_position => "beginning"
    }
    
    # 通過給日志事件定義類型來區分
    file {
        path => ["/var/log/nginx/error.log"]
        type => "nginx_error"
        start_position => "beginning"
    }

    # 通過給日志事件新增字段來區分
    file {
        path => ["/var/log/nginx/api.log"]
        add_field => {"myid" => "api"}
        start_position => "beginning"
    }
}

filter {
    # 判斷類型后，分別對事件做相應處理
    if [type] == "nginx_access" {
        grok {
            match => { "message" => "" }
        }
    }

    if [type] == "nginx_error" {
        grok {
            match => { "message" => "" }
        }
    }

    if [myid] == "api" {
        grok {
            match => { "message" => "" }
        }
    }
}

output {
    # 根據類型的不同，分別存儲到不同的索引名稱中
    if [type] == 'nginx_access' {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "logstash_access-%{+YYYY.MM.dd}"
        }
    }

    if [type] == 'nginx_error' {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "logstash_error-%{+YYYY.MM.dd}"
        }
    }

    if [myid] == "api" {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "logstash_api-%{+YYYY.MM.dd}"
        }
    }
}

 

2、以redis作為輸入源

input {
    redis {
        host => '10.105.199.10'
        type => 'web_error'
        port => '8000'
        data_type => 'list'
        key => 'web_error'
        password => "E1e7ed7eF437416165597b956fac004e"
        db => 0
    }

}

output {
    if [type] == "web_error" {
        elasticsearch {
            hosts => ["127.0.0.1:9200"]
            index => "logstash_web_error-%{+YYYY.MM.dd}"
        }
    }

}

 

 3、以kafka作為輸入源

input {
    kafka {
        bootstrap_servers => "10.105.199.10:9092"
        topics => ["www.example.com"]
        codec => "json"
    }
}

filter {
    grok {
        match => {
            "message" => "正則表達式匹配nginx日志"
        }
    }
}

output {
    elasticsearch {
        hosts => ["127.0.0.1:9200"]
        index => "logstash-www.example.com_%{+YYYY.MM.dd}"
    }
}