View difference between Paste ID: SNRumeBN and V6fqS883
SHOW: | | - or go back to the newest paste.
1
# Where to get input
2
input {
3
  # syslog inputs
4
  tcp {
5
    port => 5000
6
    type => "syslog"
7
  }
8
  udp {
9
    port => 5000
10
    type => "syslog"
11
  }
12
13
  # CoreOS journal input
14
  tcp {
15
    codec => "json_lines"
16
    port => 5004
17
    tags => ["coreos","docker"]
18
    type => "systemd"
19
  }
20
21
  # Logspout input
22
  tcp {
23
    codec => "json_lines"
24
    port => 5006
25
    tags => ["docker"]
26
    type => "logspout"
27
  }
28
29
  # Log4j application input
30
  log4j {
31
    codec => "json_lines"
32
    port  => 4560
33
    tags  => ["applogs"]
34
    type  => "log4j"
35
  }
36
}
37
38
# Some Filtering
39
filter {
40
  # syslog filter
41
  if [type] == "syslog" {
42
    grok {
43
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
44
      add_field => [ "received_at", "%{@timestamp}" ]
45
      add_field => [ "received_from", "%{host}" ]
46
    }
47
    syslog_pri { }
48
    date {
49
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
50
    }
51
52
    if !("_grokparsefailure" in [tags]) {
53
      mutate {
54
        replace => [ "message", "%{syslog_message}" ]
55
      }
56
57
      mutate {
58
        remove_field => [  "syslog_message" ]
59
      }
60
    }
61
62
    # Remove spurious fields that have names changed or been aggregated
63
    mutate {
64
      remove_field => [ "syslog_hostname", "syslog_timestamp" ]
65
    }
66
  }
67
68
  # systemd/journal filter (CoreOS)
69
  if [type] == "systemd" {
70
    mutate { rename => [ "MESSAGE", "message" ] }
71
    mutate { rename => [ "_SYSTEMD_UNIT", "program" ] }
72
  }
73
74
  # Docker filter
75
  if [tags] == "docker" {
76
    json {
77
      source => "message"
78
    }
79
    mutate {
80
      rename => [ "log", "message" ]
81
    }
82
    date {
83
      match => [ "time", "ISO8601" ]
84
    }
85
  }
86
}
87
88
# Where to send output
89
output {
90
  # Send output to standard output device/interface
91
  stdout {
92
    codec => rubydebug
93
  }
94
95
  # Parse failed messages to separate index
96
  if "_grokparsefailure" in [tags] {
97
    elasticsearch {
98
    # host => ["localhost:9200"]
99
      host => ["ES_CONN_STR"]
100
      index => "parse-err-%{+YYYY.MM.dd}"
101
      protocol  => "http"
102
    }
103
  }
104
105
# Elasticsearch output
106
  elasticsearch {
107
  # host => ["localhost:9200"]
108
    host => ["ES_CONN_STR"]
109-
    index => "logstash-%{+YYYY.MM.dd}"
109+
    index => "cgidev-logstash-%{+YYYY.MM.dd}"
110
    protocol  => "http"
111
    user => logstash
112
    password => logstash
113
  }
114
}