View difference between Paste ID: 3eK9ByBS and QSeSgnDL
SHOW: | | - or go back to the newest paste.
1
# Where to get input
2
input {
3
  # syslog inputs
4
  tcp {
5
    port => 5000
6
    type => "syslog"
7
  }
8
  udp {
9
    port => 5000
10
    type => "syslog"
11
  }
12
13-
  # CoreOS journal input
13+
  # NAGIOS input
14
  tcp {
15
    port => 5044
16-
    port => 5004
16+
    tags => ["nagios"]
17-
    tags => ["coreos","docker"]
17+
    type => "nagios"
18-
    type => "systemd"
18+
19
20
  # Logspout input
21
  tcp {
22
    codec => "json_lines"
23
    port => 5006
24
    tags => ["docker"]
25
    type => "logspout"
26
  }
27
28
  # Log4j application input
29
  log4j {
30
    codec => "json_lines"
31
    port  => 4560
32
    tags  => ["applogs"]
33
    type  => "log4j"
34
  }
35
}
36
37
# Some Filtering
38
filter {
39
  # syslog filter
40
  if [type] == "syslog" {
41
    grok {
42
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
43
      add_field => [ "received_at", "%{@timestamp}" ]
44
      add_field => [ "received_from", "%{host}" ]
45
    }
46
    syslog_pri { }
47
    date {
48
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
49
    }
50
51
    if !("_grokparsefailure" in [tags]) {
52
      mutate {
53
        replace => [ "message", "%{syslog_message}" ]
54
      }
55
56
      mutate {
57
        remove_field => [  "syslog_message" ]
58
      }
59
    }
60
61
    # Remove spurious fields that have names changed or been aggregated
62
    mutate {
63
      remove_field => [ "syslog_hostname", "syslog_timestamp" ]
64
    }
65
  }
66
67
  # systemd/journal filter (CoreOS)
68
  if [type] == "systemd" {
69
    mutate { rename => [ "MESSAGE", "message" ] }
70
    mutate { rename => [ "_SYSTEMD_UNIT", "program" ] }
71
  }
72
73
  #Nagios filter
74
  if [type] == "nagios" {
75
    grok {
76
      match => { "message" => "%{NAGIOSLOGLINE}" }
77
    }
78
  }
79
   
80
  # Docker filter
81
  if [tags] == "docker" {
82
    json {
83
      source => "message"
84
    }
85
    mutate {
86
      rename => [ "log", "message" ]
87
    }
88
    date {
89
      match => [ "time", "ISO8601" ]
90
    }
91
  }
92
}
93
94
# Where to send output
95
output {
96
  # Send output to standard output device/interface
97
  stdout {
98
    codec => rubydebug
99
  }
100
101
  # Parse failed messages to separate index
102
  if "_grokparsefailure" in [tags] {
103
    elasticsearch {
104
    # host => ["localhost:9200"]
105
      host => ["ES_CONN_STR"]
106
      index => "cgidev-parse-err-%{+YYYY.MM.dd}"
107
      protocol  => "http"
108
      user => logstash
109
      password => logstash
110
    }
111
  }
112
113
# Elasticsearch output
114
  elasticsearch {
115
  # host => ["localhost:9200"]
116
    host => ["ES_CONN_STR"]
117
    index => "cgidev-logstash-%{+YYYY.MM.dd}"
118
    protocol  => "http"
119
    user => logstash
120
    password => logstash
121
  }
122
}