docker部署logstash

1、下載鏡像

[root@vanje-dev01 ~]# docker pull logstash:7.0.1

 

2、安裝部署

  2.1  創建宿主映射目錄

# mkdir /etc/logstash/

# mkdir /etc/logstash/conf.d

  2.2 編輯配置文件

# vim /etc/logstash/logstash.yml

 http.host: "0.0.0.0"

# touch /etc/logstash/patterns

# chown -R 1000:1000 /etc/logstash/

  2.3 啟動

# docker run -d --name logstash \

-p 5044:5044 -p 9600:9600 \

--net=host --restart=always \

-v /home/logstash:/usr/share/logstash \

logstash:7.1.1

目錄映射說明：

　　1、/etc/logstash/logstash.yml：logstash啟動配置文件

　　2、/etc/logstash/conf.d： logstash過濾日志規則

　　3、/etc/logstash/patterns： 自定義匹配規則

生產中為了保證elk集群的穩定性，Logstash作為日志的解析分割，一般都會在前面添加redis或者kafka集群。

 

springcloud+logback+logstash+rabbitmq

 

pom

        <!--logback日志寫入rabbitmq-->

        <dependency>

            <groupId>org.springframework.amqp</groupId>

            <artifactId>spring-rabbit</artifactId>

        </dependency>

        <dependency>

            <groupId>org.springframework.boot</groupId>

            <artifactId>spring-boot-starter-amqp</artifactId>

        </dependency>

 

logback-spring.xml配置

 

<?xml version="1.0" encoding="UTF-8"?>

<configuration debug="false">

<!--定義日志文件的存儲地址 -->

<property name="LOG_HOME" value="../logs/system" />

 

<!--<property name="COLOR_PATTERN" value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta( %replace(%caller{1}){'\t|Caller.{1}0|\r\n', ''})- %gray(%msg%xEx%n)" />-->

<!-- 控制台輸出 -->

<!--<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">-->

<!-- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">-->

<!-- <!–格式化輸出：%d表示日期，%thread表示線程名，%-5level：級別從左顯示5個字符寬度%msg：日志消息，%n是換行符-->

<!-- <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>–>-->

<!-- <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n</pattern>-->

<!-- </encoder>-->

<!--</appender>-->

 

<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">

<layout class="ch.qos.logback.classic.PatternLayout">

<pattern>%highlight(%d{yyyy-MM-dd HH:mm:ss.SSS}) %boldYellow([%thread]) %-5level %logger{36}- %msg%n</pattern>

</layout>

</appender>

 

 

<!-- 按照每天生成日志文件 -->

<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

<!--日志文件輸出的文件名 -->

<FileNamePattern>${LOG_HOME}/jeecgboot-system-%d{yyyy-MM-dd}.log</FileNamePattern>

<!--日志文件保留天數 -->

<MaxHistory>7</MaxHistory>

</rollingPolicy>

<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

<!--格式化輸出：%d表示日期，%thread表示線程名，%-5level：級別從左顯示5個字符寬度%msg：日志消息，%n是換行符 -->

<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>

</encoder>

</appender>

 

<!-- 生成 error html格式日志開始 -->

<appender name="ERROR_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">

<filter class="ch.qos.logback.classic.filter.ThresholdFilter">

<!--設置日志級別,過濾掉info日志,只輸入error日志-->

<level>ERROR</level>

</filter>

<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

<!--<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">-->

<!--格式化輸出：%d表示日期，%thread表示線程名，%-5level：級別從左顯示5個字符寬度%msg：日志消息，%n是換行符 -->

<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>

<!--<layout class="ch.qos.logback.classic.html.HTMLLayout">-->

<!-- <pattern>%p%d%msg%M%F{32}%L</pattern>-->

<!--</layout>-->

 

</encoder>

<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

<!--日志文件輸出的文件名 -->

<FileNamePattern>${LOG_HOME}/error-log-%d{yyyy-MM-dd}.log</FileNamePattern>

<!--<!–日志文件保留天數 –>-->

<MaxHistory>7</MaxHistory>

</rollingPolicy>

<!--<file>${LOG_HOME}/error-log.html</file>-->

 

</appender>

 

    <!--logback日志寫入rabbitmq -->

    <appender name="RabbitMq"

              class="org.springframework.amqp.rabbit.logback.AmqpAppender">

        <layout>

            <pattern><![CDATA[ %d %p %t [%c] - <%m>%n ]]></pattern>

        </layout>

        <!--rabbitmq地址 -->

        <addresses>192.168.0.20:5670</addresses>

        <abbreviation>36</abbreviation>

        <includeCallerData>true</includeCallerData>

        <applicationId>system</applicationId>

        <username>admin</username>

        <password>admin</password>

        <!--路郵件 -->

        <!--{applicationId}  -->

        <!--%property{applicationId}.%c.%p 收集不到日志  -->

        <!--最后改為精確匹配了  非常重要 -->

        <routingKeyPattern>system</routingKeyPattern>

        <generateId>true</generateId>

        <charset>UTF-8</charset>

        <durable>true</durable>

        <deliveryMode>NON_PERSISTENT</deliveryMode>

        <declareExchange>true</declareExchange>

        <autoDelete>false</autoDelete>

    </appender>

 

    <!--myibatis log configure -->

    <logger name="com.apache.ibatis" level="TRACE" />

    <logger name="java.sql.Connection" level="DEBUG" />

    <logger name="java.sql.Statement" level="DEBUG" />

    <logger name="java.sql.PreparedStatement" level="DEBUG" />

 

    <!-- 日志輸出級別 -->

    <root level="INFO">

        <appender-ref ref="STDOUT" />

        <appender-ref ref="FILE" />

        <appender-ref ref="ERROR_LOG" />

        <!--這塊也要添加-->

        <appender-ref ref="RabbitMq" />

    </root>

 

</configuration>

 

 

Logstash 配置

 

vi /home/logstash/config/conf.d/system-log.conf

 

input {

   rabbitmq {

       type =>"all"

       durable => true

       exchange => "logs"

           exchange_type => "topic"

           key => "system"

           host => "192.168.0.20:5670"

#          port => 5670

           user => "admin"

           password => "admin"

           queue => "system-mq"

       auto_delete => false

    

  }

 }

 

output {

  elasticsearch {

       hosts => ["192.168.0.23:9200"]

           index => "system_log_mq"

           }

}