API tools faq
paste
Guest User

ELK configurations

a guest
Aug 20th, 2015
364
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
JSON 5.87 KB | None | 0 0
raw download clone embed print report
  1. # cat logstash-forwarder.conf
  2. {
  3.   "network": {
  4.     "servers": [ "192.168.3.52:5000" ],
  5.     "ssl ca": "C:\\Program Files (x86)\\logstash-forwarder-justmara\\conf\\logstash-forwarder.crt",
  6.     "timeout": 15
  7.   },
  8.  
  9.   "files": [
  10.     {
  11.       "paths": [
  12.         "C:\\tmp\\ak2csimsv51\\dialer.log"
  13.       ],
  14.       "fields": { "type": "ak2csimsv51" }
  15.     }
  16.   ]
  17. }
  18.  
  19. # cat logstash-shipper.conf
  20. input {
  21.     lumberjack {
  22.        port => 5000
  23.        ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
  24.        ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
  25.     }
  26. }
  27. filter {
  28.     if [message] == "" {
  29.         drop { }
  30.     }
  31.     grok {
  32.         match => {
  33.             "file" => "(?<log_file>[^\\\/]+)\.log$"
  34.         }
  35.     }
  36. }
  37. output {
  38.     redis { host => "127.0.0.1" data_type => "list" key => "logstash" }
  39.     file {
  40.         message_format => "%{[@timestamp]} %{[_id]}: %{[message]}"
  41.         codec => "json"
  42.         path => "/home/user/Documents/logstash/file-output/fromshipper.log"
  43.     }
  44. }
  45.  
  46. # cat 002-redis-input.conf
  47. input {
  48.   redis {
  49.     host => "127.0.0.1"
  50.     data_type => "list"
  51.     key => "logstash"
  52.     codec => json
  53.   }
  54. }
  55.  
  56. # cat 101-filter.conf
  57. filter {
  58.     # type 1 - json - multiline: LOGLEVEL TIMESTAMP DATA
  59.     if [log_file] in ["activityserver","agentserver","dialer","ppserver","router","rtdataprovider","rtpprocessor","scenarioengine","screenrecorder","sipprocessor","smsserver","statserver"] {
  60.         mutate {
  61.             add_field => {
  62.                 "format" => "101 - json"
  63.                 "received_at" => "%{@timestamp}"
  64.             }
  65.         }
  66.         multiline {
  67.             #allow_duplicates => boolean
  68.             max_age => 60 # in seconds
  69.             negate => true
  70.             pattern => "^%{CSIM_LOGLVL}\s+%{CSIM_TIMESTAMP}"
  71.             #patterns_dir => array
  72.             periodic_flush => true #boolean
  73.             #source => string
  74.             stream_identity => "%{host}.%{file}.%{type}" # string
  75.             what => previous
  76.         }
  77.         grok {
  78.             match => {
  79.                 "message" => "%{CSIM_LOGLVL:log_level} %{CSIM_TIMESTAMP:[@metadata][timestamp]} %{GREEDYDATA:body}"
  80.             }
  81.         }
  82.         date {
  83.             timezone => "Asia/Tokyo"
  84.             match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
  85.         }
  86.     }
  87. }
  88.  
  89. # cat 103-filter.conf
  90. filter {
  91.     # type3 - request singleline:
  92.     #Format:
  93.     #Timestamp Body;
  94.     #Body: ClientIP "Verb Request  HTTP/1.1" Response Bytes
  95.     if [log_file] in ["ad-request","clientweb-request","crmserver-request","portal-request"] {
  96.         mutate {
  97.             add_field => {
  98.                 "format" => "103 - request"
  99.                 "received_at" => "%{@timestamp}"
  100.             }
  101.         }
  102.         grok {
  103.             named_captures_only => true
  104.             match => {
  105.                 "message" => "%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>%{IP:clientip} \"%{WORD:verb} %{URIPATHPARAM:request} HTTP\/(?<http_version>\d\.\d)\" %{NUMBER:response:int} %{NUMBER:bytes:int})"
  106.             }
  107.         }
  108.         date {
  109.             #timezone => "America/Los_Angeles"
  110.             #timezone => "Europe/Kiev"
  111.             timezone => "Asia/Tokyo"
  112.             match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
  113.         }
  114.     }
  115. }
  116.  
  117. # 104-filter.conf
  118. filter {
  119.     # type 4.1 - scanner - attention there are multiline:
  120.     #Format:
  121.     # Loglvl Timestamp Body;
  122.     # Body: [logger] who - what
  123.     if [log_file] in ["ad-server","clientweb-server","crm-server","portal-server","ad-client","portal-client","aggregator","cfgsrv","dbsrv","emailserver","scheduler"]  {
  124.         mutate {
  125.             add_field => {
  126.                 "format" => "104 - server"
  127.                 "received_at" => "%{@timestamp}"
  128.             }
  129.         }
  130.         if [log_file] not in ["clientweb-server","crm-server","aggregator","dbsrv"]  {
  131.             multiline {
  132.                 #allow_duplicates => boolean
  133.                 max_age => 60
  134.                 negate => true
  135.                 pattern => "^%{CSIM_LOGLVL}\s+%{CSIM_TIMESTAMP}"
  136.                 #patterns_dir => array
  137.                 periodic_flush => true #boolean
  138.                 #source => string
  139.                 stream_identity => "%{host}.%{file}.%{type}" # string
  140.                 what => previous
  141.             }
  142.         }
  143.         grok {
  144.             match => {
  145.                 #"message" => "%{CSIM_LOGLVL:log_level}\s+%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>\[(?<logger>.+)\] (?<who>.+?)\s+-\s+(?<what>.*))"
  146.                 "message" => "%{CSIM_LOGLVL:log_level}\s+%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>.*)"
  147.             }
  148.         }
  149.         date {
  150.             #timezone => "America/Los_Angeles"
  151.             #timezone => "Europe/Kiev"
  152.             timezone => "Asia/Tokyo"
  153.             match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
  154.         }
  155.     }
  156. }
  157.  
  158. # 105-filter.conf
  159. filter {
  160.     if [log_file] in ["ad.wrapper","clientweb.wrapper","crmserver.wrapper","portal.wrapper"] {
  161.         mutate {
  162.             add_field => {
  163.                 "format" => "105 - wrapper"
  164.                 "received_at" => "%{@timestamp}"
  165.             }
  166.         }
  167.         grok {
  168.             break_on_match => false
  169.             match => {
  170.                 "message" => "%{CSIM_LOGLVL:log_level}\|(?<logger>.+?)\|%{TIMESTAMP_ISO8601:[@metadata][timestamp]}\|%{GREEDYDATA:body}"
  171.             }
  172.         }
  173.         date {
  174.             #timezone => "America/Los_Angeles"
  175.             #timezone => "Europe/Kiev"
  176.             timezone => "Asia/Tokyo"
  177.             match => [ "[@metadata][timestamp]" , "yy-MM-dd HH:mm:ss" ]
  178.         }
  179.     }
  180. }
  181.  
  182. # cat 107-test_label-filter.conf
  183. filter {
  184.         mutate {
  185.             add_field => { "test_label" => "stress_test_dialer" }
  186.         }
  187.  
  188. }
  189.  
  190. # cat 202-elasticsearch-output.conf
  191. output {
  192.     if [type] != "load-test" {
  193.         elasticsearch {
  194.             host => "localhost"
  195.         }
  196.     }
  197. }
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!