######### application.properties
# trace and span taken from MDC context. Hostname could be redundant is using FileBeat on same host,
# but maybe you are feeding logs to ES in another way. Including it in the log covers this possibility

logging.pattern.file=%d{ABSOLUTE} [%X{traceId}-%X{spanId}] %-5p ${HOSTNAME} ${PID} [%t] [%C{2}] [%F:%L] - %m%n



#########  pipeline/logstash.conf
# There is an old saying. You had a problem. You solved it with regex. Now you have 2 problems.
# Really, you can address this is a simpler way by using || or [] separators between each field,
# but anyhow, where's the fun in that.
# Note that if you do not add the date mutation, the timestamp in ES will be the time it was received from the FileBeat
# as opposed to the timestamp of the log message. And that is bad.
input {
  beats {
    port => 5044
  }
}
filter {
  grok {
    match => [ "message",
               "(?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME}) \[(?<traceid>[a-f0-9]*)-(?<spanid>[a-f0-9]*)\] %{LOGLEVEL:level}\s+(?<hostname>[a-zA-Z0-9.-]*)\s+%{NUMBER:pid}\s*\[(?<thread>[A-Za-z0-9-]+)\]\s*\[[A-Za-z0-9.]*\.(?<class>[A-Za-z0-9#_]+)\] \[(?<file>[A-Za-z0-9.]+)\:(?<line>[0-9]+)\]\s*\-\s*(?<logmessage>.*)"             ]
  }
  date {
                  locale => "en"
                  timezone => "UTC"
                  match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
                  target => "@timestamp"
                  remove_field => ["timestamp", "monthday", "year", "month", "day", "time"]
  }
}
output {

  elasticsearch {
    hosts => ["elasticsearch:9200"]
    user => elastic
    password => changeme
    index => "logstash-%{+YYYY.MM.dd}"
  }
  stdout {
    codec => rubydebug
  }
}