View difference between Paste ID: B0hdRP3m and 3eK9ByBS
SHOW: | | - or go back to the newest paste.
1
# Where to get input
2
input {
3
  # syslog inputs
4
  tcp {
5
    port => 5000
6
    type => "syslog"
7
  }
8
  udp {
9
    port => 5000
10
    type => "syslog"
11
  }
12
13
  # NAGIOS input
14
  tcp {
15
    port => 5044
16
    ssl => false
17
    tags => ["nagios"]
18
    type => "nagios"
19
  }
20
21
  # Logspout input
22
  tcp {
23
    codec => "json_lines"
24
    port => 5006
25
    tags => ["docker"]
26
    type => "logspout"
27
  }
28
29
  # Log4j application input
30
  log4j {
31
    codec => "json_lines"
32
    port  => 4560
33
    tags  => ["applogs"]
34
    type  => "log4j"
35
  }
36
}
37
38
# Some Filtering
39
filter {
40
  # syslog filter
41
  if [type] == "syslog" {
42
    grok {
43
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
44
      add_field => [ "received_at", "%{@timestamp}" ]
45
      add_field => [ "received_from", "%{host}" ]
46
    }
47
    syslog_pri { }
48
    date {
49
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
50
    }
51
52
    if !("_grokparsefailure" in [tags]) {
53
      mutate {
54
        replace => [ "message", "%{syslog_message}" ]
55
      }
56
57
      mutate {
58
        remove_field => [  "syslog_message" ]
59
      }
60
    }
61
62
    # Remove spurious fields that have names changed or been aggregated
63
    mutate {
64
      remove_field => [ "syslog_hostname", "syslog_timestamp" ]
65
    }
66
  }
67
68
  # systemd/journal filter (CoreOS)
69
  if [type] == "systemd" {
70
    mutate { rename => [ "MESSAGE", "message" ] }
71
    mutate { rename => [ "_SYSTEMD_UNIT", "program" ] }
72
  }
73
74
  #Nagios filter
75
  if [type] == "nagios" {
76
    grok {
77
      match => { "message" => "%{NAGIOSLOGLINE}" }
78
    }
79
  }
80
   
81
  # Docker filter
82
  if [tags] == "docker" {
83
    json {
84
      source => "message"
85
    }
86
    mutate {
87
      rename => [ "log", "message" ]
88
    }
89
    date {
90
      match => [ "time", "ISO8601" ]
91
    }
92
  }
93
}
94
95
# Where to send output
96
output {
97
  # Send output to standard output device/interface
98
  stdout {
99
    codec => rubydebug
100
  }
101
102
  # Parse failed messages to separate index
103
  if "_grokparsefailure" in [tags] {
104
    elasticsearch {
105
    # host => ["localhost:9200"]
106
      host => ["ES_CONN_STR"]
107
      index => "cgidev-parse-err-%{+YYYY.MM.dd}"
108
      protocol  => "http"
109
      user => logstash
110
      password => logstash
111
    }
112
  }
113
114
# Elasticsearch output
115
  elasticsearch {
116
  # host => ["localhost:9200"]
117
    host => ["ES_CONN_STR"]
118
    index => "cgidev-logstash-%{+YYYY.MM.dd}"
119
    protocol  => "http"
120
    user => logstash
121
    password => logstash
122
  }
123
}