|
ELK之filebeat收集多类型日志 1.IP规划 10.0.0.33:filebeat+tomcat,filebeat收集系统日志、tomcat日志发送到logstash 10.0.0.32:logstash,将日志写入reids(input、output) 10.0.0.31:redis,大量缓存数据 10.0.0.30:logstash,从redis取出数据写入es(input、output) 10.0.0.29:es+kibana,es接收传来的数据写入磁盘,等待kibana来取 a.10.0.0.33:filebeat输出到logstash vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log paths: - /var/log/.log - /var/log/messages exclude_lines: [’^DBG’,"^$"] document_type: filebeat-systemlog-0033 - input_type: log paths: - /usr/local/tomcat/logs/tomcat_access_log..log exclude_lines: [’^DBG’,"^$"] document_type: tomcat-accesslog-0033 output.logstash: hosts: [“10.0.0.32:5044”] enabled: true worker: 2 compression_level: 3 # systemctl restart filebeat b.10.0.0.32:logstash将日志写入reids(向redis写数据不需要给key加日期) vim beats.conf input { beats { port => “5044” } } output { if [type] == “filebeat-systemlog-0033” { redis { data_type => “list” host => “10.0.0.31” db => “3” port => “6379” password => “123456” key => “filebeat-systemlog-0033” } } if [type] == “tomcat-accesslog-0033” { redis { data_type => “list” host => “10.0.0.31” db => “4” port => “6379” password => “123456” key => “tomcat-accesslog-0033” } } } # systemctl restart logstash c.10.0.0.31:redis不用做什么操作 d.10.0.0.30:logstash从redis取出数据写入es vim redis-es.conf input { redis { data_type => “list” host => “10.0.0.31” db => “3” port => “6379” key => “filebeat-systemlog-0033” password => “123456” } redis { data_type => “list” host => “10.0.0.31” db => “4” port => “6379” key => “tomcat-accesslog-0033” password => “123456” } } output { if [type] == “filebeat-systemlog-0033” { elasticsearch { hosts => [“10.0.0.29:9200”] index => “redis31-systemlog-%{+YYYY.MM.dd}” } } if [type] == “tomcat-accesslog-0033” { elasticsearch { hosts => [“10.0.0.29:9200”] index => “tomcat-accesslog-0033-%{+YYYY.MM.dd}” } } } # systemctl restart logstash e.10.0.0.29:es+kibana es插件页面出现这个日志索引时tomcat-accesslog-0033-xxxx.xx.xx,代表整个流程是通的. (责任编辑：IT) |