logstash插件推荐

1、kafka
参考:https://www.elastic.co/guide/en/logstash/current/plugins-inputs-kafka.html
2、hdfs
参考:https://www.elastic.co/guide/en/logstash/5.4/plugins-outputs-webhdfs.html
3、zabbix
参考:https://www.elastic.co/guide/en/logstash/5.4/plugins-outputs-zabbix.html

input插件
https://www.elastic.co/guide/en/logstash/5.4/input-plugins.html
output插件
https://www.elastic.co/guide/en/logstash/5.4/output-plugins.html

logstash之input配置collectd类型详解

配置
input {
    udp {
        port => 12000
        codec => collectd {}
        type => "collectd-demo"
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

启动
bin/logstash -f /etc/logstash/conf.d/demo-input-collectd.conf
结果
{
         "@timestamp" => 2017-06-13T03:20:19.620Z,
    "plugin_instance" => "root",
      "type_instance" => "free",
             "plugin" => "df",
               "host" => "192.168.56.201",
           "@version" => "1",
      "collectd_type" => "df_complex",
               "type" => "collectd-demo",
              "value" => 5521645568.0
}
{
       "@timestamp" => 2017-06-13T03:20:19.620Z,
           "plugin" => "entropy",
             "host" => "192.168.56.201",
         "@version" => "1",
    "collectd_type" => "entropy",
             "type" => "collectd-demo",
            "value" => 844.0
}

logstash之input配置redis类型详解

用途
监控redis数据
配置示例
input {
    redis {
        data_type => "list"
        key => "logstash-demo"
        host => "127.0.0.1"
        port => 6379
        threads => 5
    }
}
output {
 stdout {
 codec => rubydebug
 }
}

启动
bin/logstash -f /etc/logstash/conf.d/demo-input-redis.conf

测试
redis-cli -h 127.0.0.1
rpush logstash-demo test
结果
{
    "@timestamp" => 2017-06-12T13:55:11.689Z,
      "@version" => "1",
       "message" => "test",
          "tags" => [
        [0] "_jsonparsefailure"
    ]
}
date_type	只能是list(使用BLPOP获取消息)、channel(使用SUBSCRIBE获取消息)、pattern_channel(使用PSUBSCRIBE获取消息)

logstash之input配置syslog类型详解

用途
监控syslog,监控系统运行情况
配置示例
input {
    syslog {
        port => 5000
        type => "demo-syslog"
    }
}

output {
    stdout {
        codec => rubydebug
    }
}


启动
bin/logstash -f /etc/logstash/conf.d/demo-input-syslog.conf

测试
telnet localhost 5000
结果
{
          "severity" => 0,
        "@timestamp" => 2017-06-12T09:41:46.655Z,
          "@version" => "1",
              "host" => "127.0.0.1",
           "message" => "heloooooooo\r\n",
              "type" => "demo-syslog",
          "priority" => 0,
          "facility" => 0,
    "severity_label" => "Emergency",
              "tags" => [
        [0] "_grokparsefailure_sysloginput"
    ],
    "facility_label" => "kernel"
}

logstash之input配置stdin类型详解

配置示例
input {
	stdin {
		type => "demo-stdin"
		add_field => {"test" => "hello"}
        codec => "plain"
		tags => ["stdin-test"]
	}
}
output {
	stdout{
		codec=>rubydebug
	}
}
启动
bin/logstash -f /etc/logstash/conf.d/demo-input-stdin.conf
输入test
返回
{
    "@timestamp" => 2017-06-12T07:39:40.278Z,
          "test" => "hello",
      "@version" => "1",
          "host" => "192-168-56-201",
       "message" => "test",
          "type" => "demo-stdin",
          "tags" => [
        [0] "stdin-test"
    ]
}

logstash之input配置file类型详解

监听文件变化,记录一个.sincedb的数据库文件跟踪监听文件读取位置(记录的是时间戳)

配置案例
input {
 file {
 path => ["/data/test/test/*.log"]
 type => "demo-log"
 start_position => "beginning"
 }
}
output {
 stdout{
 codec=>rubydebug
 }
}
检查配置
bin/logstash -f /etc/logstash/conf.d/demo.conf -t
启动
bin/logstash -f /etc/logstash/conf.d/demo.conf
测试
在/data/test/test/目录建立1.log文件,然后执行
echo `date` >> /data/test/test/1.log
观察
{
          "path" => "/data/test/test/1.log",
    "@timestamp" => 2017-06-12T06:43:15.742Z,
      "@version" => "1",
          "host" => "192-168-56-201",
       "message" => "Mon Jun 12 06:43:15 UTC 2017",
          "type" => "demo-log"
}

配置说明
path 	监听文件绝对路径,可以使用字符串或数组
excude 	排除不监听的文件,可以使用字符串或者数组
type 	事件添加分类字段 
start_position 	只能是"beginning"或者"end",默认是end,end类似tail -f;beginning是从文件开头开始(导入历史数据)
stat_interval 	每隔多久检查一次文件变化,默认1s
add_field		新增一个字段
tags			用于增加一些标签,这个标签可能在后续的处理中起到标志的作用
delimiter		分行标识
discover_interval 每隔多久检查是否有新文件,默认15s
close_older		如果监听的文件达到设置的时间内未改动则关闭文件句柄,默认3600
ignore_older	在每次检查文件列表的时候,如果一个文件的最后修改时间超过这个值,就忽略这个文件。默认是 86400 秒
sincedb_path	记录读取文件位置
sincedb_write_interval	每隔多久写入读取文件位置信息,默认15s