input{
  file{
    type=>"python"
    path=>["/home/test/logs/*"]
    exclude=>['*.tar', '*.tgz', '*.gz', '*.bz2', '*.zip']
    start_position=>"end"
  }
  file{
    type=>"system_messages"
    path=>["/var/log/messages*"]
    exclude=>["*.tar","*.tgz","*.gz","*.bz2","*.zip"]
    start_position=>"end"
  }
  file{
    type=>"system_cron"
    path=>["/var/log/cron*"]
    exclude=>["*.tar","*.tgz","*.gz","*.bz2","*.zip"]
    start_position=>"end"
  }
  file{
    type=>"system_yum"
    path=>["/var/log/yum.log*"]
    exclude=>["*.tar","*.tgz","*.gz","*.bz2","*.zip"]
    start_position=>"end"
  }
  file{
    type=>"nginx_access"
    path=>["/var/log/nginx/*access.log*"]
    exclude=>["*.tar","*.tgz","*.gz","*.bz2","*.zip"]
    start_position=>"end"
  }
  file{
    type=>"nginx_error"
    path=>["/var/log/nginx/*error.log*"]
    exclude=>["*.tar","*.tgz","*.gz","*.bz2","*.zip"]
    start_position=>"end"
  }
}

filter{
  if [type] == "python" {
    multiline{
      pattern => "^\d{4}"
      negate => true
      what => "previous"
    }
    if [type] == "tomcat" {
      grok{
        match=>{ "message" => ".?(?<datetime>\d{4}.\d{2}.\d{2} \d{2}.\d{2}.\d{2}(.\d{3})?).? \[(?<level>.*?),(?<logger_name>.*?),(?<thread_name>.*?),(?<assetType>.*?),(?<requestId>.*?),(?<assetId>.*?)\] %{GREEDYDATA}" }
      }
    }
    if [type] == "nodejs" {
      grok{
        match=>{ "message" => ".?(?<datetime>\d{4}.\d{2}.\d{2} \d{2}.\d{2}.\d{2}(.\d{3})?).? \[(?<level>.*?)\] %{GREEDYDATA}" }
      }
    }
    if [type] == "python" {
      grok{
        match=>{ "message" => ".?(?<datetime>\d{4}.\d{2}.\d{2} \d{2}.\d{2}.\d{2}(.\d{3})?).? \[(?<level>.*?),%{GREEDYDATA}" }
      }
    }
    if [type] == "jenkins" {
      grok{
        match=>{ "message" => ".?(?<datetime>\d{4}.\d{2}.\d{2} \d{2}.\d{2}.\d{2}(.\d{3})?).? %{GREEDYDATA}" }
      }
    }
  } else if [type] =~ "system_" {
    grok{
      match=>{ "message" => "(?<datetime>\w{3} +\d{1,2} \d{2}:\d{2}:\d{2}) %{GREEDYDATA}" }
    }
  } else if [type] == "nginx_access" {
    grok{
      match=>{ "message" => ".*(?<datetime>\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}) %{GREEDYDATA}" }
    }
  } else if [type] == "nginx_error" {
    grok{
      match=>{ "message" => "(?<datetime>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) %{GREEDYDATA}" }
    }
  }
  date{
    match => [ "datetime", "YYYY-MM-dd HH:mm:ss", "YYYY-MM-dd HH:mm:ss.SSS", "YYYY-MM-dd HH:mm:ss,SSS", "YYYY/MM/dd HH:mm:ss", "YYYY/MM/dd HH:mm:ss.SSS", "YYYY/MM/dd HH:mm:ss,SSS", "YYYY.MM.dd HH:mm:ss", "YYYY.MM.dd HH:mm:ss.SSS", "YYYY.MM.dd HH:mm:ss,SSS", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "dd/MMM/YYYY:HH:mm:ss" ]
  }
  alter{
    add_field => {
      "service" => "test"
    }
    remove_field => [ "datetime" ]
  }
  if [level] == "ERROR" {
    throttle {
        before_count => -1
        after_count => 1
        period => 3600
        key => "%{service}%{host}"
        add_tag => "throttled"
    }
  }
}

output{
  if [level] == "ERROR" and "throttled" not in [tags] {
    hipchat{
      room_id => "xxxxx"
      token => "xxxxx"
      format => "[%{service}][%{host}] %{message}"
    }
  }
}