Logstash 日志采集工具

    Logstash是一个开源的用于收集，分析和存储日志的工具。

    Logstash: Logstash服务的组件，用于处理传入的日志。不过是基于Elasticsearch配置使用。

    Elasticsearch: 存储所有日志。

    做个示例监控test-http、tomcat和test-api等系统的运行状态，输出error信息到elasticsearch。需要在每一台test服务器上安装配置。

    一、下载

logstash：https://download.elastic.co/logstash/logstash/logstash-2.4.1.zip

unzip logstash-2.4.1.zip

    二、log4j的配置

### 设置###

log4j.rootLogger = debug,stdout,D,E

### 输出信息到控制抬 ###

log4j.appender.stdout = org.apache.log4j.ConsoleAppender

log4j.appender.stdout.Target = System.out

log4j.appender.stdout.layout = org.apache.log4j.PatternLayout

log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n 

### 输出DEBUG 级别以上的日志到  path/logs/error.log ###

log4j.appender.D = org.apache.log4j.DailyRollingFileAppender

log4j.appender.D.File = logs/log.log

log4j.appender.D.Append = true

log4j.appender.file.ImmediateFlush=false

log4j.appender.file.BufferedIO=true

log4j.appender.file.BufferSize=8192

log4j.appender.D.Threshold = DEBUG

log4j.appender.D.layout = org.apache.log4j.PatternLayout

log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} %c [%t]-[%p] %m%n

### 输出ERROR 级别以上的日志到  path/logs/error.log ###

log4j.appender.E = org.apache.log4j.DailyRollingFileAppender

log4j.appender.E.File =logs/error.log

log4j.appender.E.Append = true

log4j.appender.E.Threshold = ERROR

log4j.appender.E.layout = org.apache.log4j.PatternLayout

log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} %c [%t]-[%p] %m%n

    三、test系统的logstash配置，注意logs目录位置和ElasticSearch的Hosts

# vim test-api.conf

input {

    file {

        path => "/opt/test-api/logs/error.log"

        start_position => "beginning"

        type => "test-api"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} %{NOTSPACE:clazz} \[%{NOTSPACE:thread-id}\]\-\[%{LOGLEVEL:level}\] %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

# cat test-http.conf

input {

    file {

        path => "/opt/test-http/logs/error.log"

        start_position => "beginning"

        type => "test"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} %{NOTSPACE:clazz} \[%{NOTSPACE:thread-id}\]\-\[%{LOGLEVEL:level}\] %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

# cat test_logs-tomcat.conf

input {

    file {

        path => "/opt/server/tomcat/logs/web.log"

        start_position => "beginning"

        type => "tomcat"

    }

}

filter {

    multiline {

        pattern => "^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}"

        negate => true

        what => "previous"

    }

    grok {

        match => { "message" => "%{TIMESTAMP_ISO8601:datetime} \[%{NOTSPACE:thread-id}\] %{LOGLEVEL:level}\s*%{NOTSPACE:clazz} \- %{GREEDYDATA:msg}" }

    }

}

output {

    elasticsearch {

        hosts => ["10.207.101.100:9200","10.207.101.101:9200","10.207.101.102:9200"]

        index => "test_logs-%{+YYYYMMdd}"

        document_type => "logs"

    }

}

设置启动

# cat  test-start.sh

nohup bin/logstash -f test-http.conf > /dev/null 2>&1 &

nohup bin/logstash -f test-api.conf > /dev/null 2>&1 &

nohup bin/logstash -f test_logs-tomcat.conf > /dev/null 2>&1 &