SpringBoot+Redis+ELK日志集中存储和查询

安装Elasticsearch#

http://www.tianch.xyz/archives/%E4%BD%BF%E7%94%A8docker-compose%E5%AE%89%E8%A3%85skywalking%E5%92%8Celasticsearchmd

增加一些配置

• 进入elasticsearch容器

  1
  docker exec -it elasticsearch bash

• 编辑config/elasticsearch.yml 追加配置

  1
  http.cors.enabled: true
  2
  http.cors.allow-origin: "*"
  3
  http.cors.allow-headers: Authorization
  4
  xpack.security.enabled: true
  5
  xpack.security.transport.ssl.enabled: true

• 重启

  1
  docker restart elasticsearch

• 设置密码

  1
  bin/elasticsearch-setup-passwords interactive

• 再重启

  1
  docker restart elasticsearch

• 设置分片-进到容器中先

  1
  curl -XPUT -u elastic -H "Content-Type:application/json" http://localhost:9200/_cluster/settings -d '{ "persistent": { "cluster": { "max_shards_per_node": 10000 } } }'

安装Kibana#

• 拉取镜像

  1
  docker pull kibana:7.6.2

• 运行容器

  1
  docker run --name kibana --link=elasticsearch --net skywalking_skywalking  -p 5601:5601 -d kibana:7.6.2

• 修改配置

  1
  docker exec -it kibana bash
  2
  cd config
  3
  vi kibana.yml

  1
  # Default Kibana configuration for docker target
  2
  server.name: kibana
  3
  server.host: "0"
  4
  elasticsearch.hosts: ["http://elasticsearch:9200"]
  5
  xpack.monitoring.ui.container.elasticsearch.enabled: true
  6
  i18n.locale: "zh-CN"
  7
  elasticsearch.username: "elastic"
  8
  elasticsearch.password: "123456"

• 重启

  1
  docker restart kibana

安装Logstash#

• 拉取镜像

  1
  docker pull logstash:7.6.2

• 编辑配置文件

  1
  mkdir -p /data/elk/logstash
  2
  vim /data/elk/logstash/logstash.yml

  1
  http.host: "0.0.0.0"
  2
  xpack.monitoring.elasticsearch.hosts: ["http://192.168.0.233:9200"]
  3
  xpack.monitoring.elasticsearch.username: "logstash_system"
  4
  xpack.monitoring.elasticsearch.password: "77589910"
  5
  #path.config: /data/elk/logstash/conf.d/*.conf
  6
  path.config: /data/docker/logstash/conf.d/*.conf
  7
  path.logs: /var/log/logstash

• 编辑conf文件 就testlog.conf

  1
  mkdir /data/elk/logstash/conf.d
  2
  vim /data/elk/logstash/conf.d/testlog.conf

  1
  input {
  2
      redis {
  3
          codec => json
  4
          data_type => "list"
  5
          key => "logstash-list"
  6
          host => "192.168.0.233"
  7
          port => "6379"
  8
          password => "123456"
  9
          batch_count => 100
  10
          threads => 1
  11
      }
  12
  }
  13
  
  14
  output {
  15
    elasticsearch {
  16
      hosts => ["192.168.0.233:9200"]
  17
      index => "platform-log-%{+YYYY.MM}"
  18
      user => "elastic"
  19
      password => "77589910"
  20
    }
  21
  }

  展开收起

• 编辑本地rsyslog配置增加

  1
  vim /etc/rsyslog.conf #增加下面一行
  2
  *.* @@192.168.0.233:5044

• 重启服务使配置生效

  1
  systemctl restart rsyslog

• 创建容器

  1
  docker run -d --restart=always --log-driver json-file --log-opt max-size=1000m --log-opt max-file=30 -p 5044:5044 --name logstash -v /data/elk/logstash/logstash.yml:/usr/share/logstash/config/logstash.yml -v /data/elk/logstash/conf.d/:/data/docker/logstash/conf.d/ logstash:7.6.2

发送日志到redis#

• 需要的依赖

  1
  <dependency>
  2
      <groupId>de.idealo.logback</groupId>
  3
      <artifactId>logback-redis</artifactId>
  4
  </dependency>
  5
  
  6
  <dependency>
  7
      <groupId>com.yomahub</groupId>
  8
      <artifactId>tlog-all-spring-boot-starter</artifactId>
  9
  </dependency>
  10
  
  11
  <dependency>
  12
      <groupId>com.yomahub</groupId>
  13
      <artifactId>tlog-logstash-logback</artifactId>
  14
  </dependency>

• 设置logback-spring.xml

  1
  <?xml version="1.0" encoding="UTF-8"?>
  2
  <configuration>
  3
  
  4
      <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
  5
          <!--这里替换成AspectLogbackEncoder-->
  6
          <encoder class="com.yomahub.tlog.core.enhance.logback.AspectLogbackEncoder">
  7
              <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern>
  8
          </encoder>
  9
      </appender>
  10
  
  11
      <!-- 获取yml中的配置 -->
  12
      <springProperty scope="context" name="redis-host" source="spring.redis.host"/>
  13
      <springProperty scope="context" name="redis-port" source="spring.redis.port"/>
  14
      <springProperty scope="context" name="redis-password" source="spring.redis.password"/>
  15
  
  16
      <!--程序关闭时关闭redis批处理的钩子-->
  17
      <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
  18
      <appender name="Logstash" class="net.logstash.logback.appender.LoggingEventAsyncDisruptorAppender">
  19
          <ringBufferSize>131072</ringBufferSize>
  20
          <appender class="de.idealo.logback.appender.RedisBatchAppender">
  21
              <connectionConfig>
  22
                  <!--redis node-->
  23
                  <scheme>NODE</scheme>
  24
                  <host>${redis-host}</host>
  25
                  <port>${redis-port}</port>
  26
                  <password>${redis-password}</password>
  27
                  <key>logstash-list</key>
  28
                  <timeout>20000</timeout>
  29
              </connectionConfig>
  30
              <maxBatchMessages>1000</maxBatchMessages>
  31
              <maxBatchSeconds>10</maxBatchSeconds>
  32
              <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
  33
                  <providers>
  34
                      <provider class="com.yomahub.tlog.logstash.logback.TLogLogstashLogbackProvider"/>
  35
                      <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符-->
  36
                      <pattern>
  37
                          <pattern>
  38
                              {
  39
                              "level": "%level",
  40
                              "thread": "%thread",
  41
                              "class": "%logger{40}",
  42
                              "message": "%message",
  43
                              "stack_trace": "%exception{10}",
  44
                              "client_time": "%d{yyyy-MM-dd HH:mm:ss.SSS}"
  45
                              }
  46
                          </pattern>
  47
                      </pattern>
  48
                      <stackTrace>
  49
                          <throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
  50
                              <maxDepthPerThrowable>10</maxDepthPerThrowable>
  51
                              <maxLength>4096</maxLength>
  52
                              <shortenedClassNameLength>20</shortenedClassNameLength>
  53
                              <rootCauseFirst>true</rootCauseFirst>
  54
                          </throwableConverter>
  55
                      </stackTrace>
  56
                  </providers>
  57
              </encoder>
  58
          </appender>
  59
      </appender>
  60
  
  61
      <appender name="Async" class="ch.qos.logback.classic.AsyncAppender">
  62
          <neverBlock>true</neverBlock>
  63
          <appender-ref ref="Logstash"/>
  64
      </appender>
  65
  
  66
      <logger name="org.springframework" level="ERROR"/>
  67
      <logger name="com.alibaba.cloud.dubbo.metadata.repository.DubboServiceMetadataRepository" level="ERROR"/>
  68
      <logger name="org.springframework.boot.autoconfigure" level="ERROR"/>
  69
      <logger name="org.springframework.data.redis" level="ERROR"/>
  70
      <logger name="com.alibaba" level="ERROR"/>
  71
      <logger name="ch.qos.logback" level="ERROR"/>
  72
      <logger name="com.tianch" level="DEBUG"/>
  73
      <root level="INFO">
  74
          <appender-ref ref="Console"/>
  75
          <appender-ref ref="Async"/>
  76
      </root>
  77
  </configuration>

  展开收起