Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # cat logstash-forwarder.conf
- {
- "network": {
- "servers": [ "192.168.3.52:5000" ],
- "ssl ca": "C:\\Program Files (x86)\\logstash-forwarder-justmara\\conf\\logstash-forwarder.crt",
- "timeout": 15
- },
- "files": [
- {
- "paths": [
- "C:\\tmp\\ak2csimsv51\\dialer.log"
- ],
- "fields": { "type": "ak2csimsv51" }
- }
- ]
- }
- # cat logstash-shipper.conf
- input {
- lumberjack {
- port => 5000
- ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
- ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
- }
- }
- filter {
- if [message] == "" {
- drop { }
- }
- grok {
- match => {
- "file" => "(?<log_file>[^\\\/]+)\.log$"
- }
- }
- }
- output {
- redis { host => "127.0.0.1" data_type => "list" key => "logstash" }
- file {
- message_format => "%{[@timestamp]} %{[_id]}: %{[message]}"
- codec => "json"
- path => "/home/user/Documents/logstash/file-output/fromshipper.log"
- }
- }
- # cat 002-redis-input.conf
- input {
- redis {
- host => "127.0.0.1"
- data_type => "list"
- key => "logstash"
- codec => json
- }
- }
- # cat 101-filter.conf
- filter {
- # type 1 - json - multiline: LOGLEVEL TIMESTAMP DATA
- if [log_file] in ["activityserver","agentserver","dialer","ppserver","router","rtdataprovider","rtpprocessor","scenarioengine","screenrecorder","sipprocessor","smsserver","statserver"] {
- mutate {
- add_field => {
- "format" => "101 - json"
- "received_at" => "%{@timestamp}"
- }
- }
- multiline {
- #allow_duplicates => boolean
- max_age => 60 # in seconds
- negate => true
- pattern => "^%{CSIM_LOGLVL}\s+%{CSIM_TIMESTAMP}"
- #patterns_dir => array
- periodic_flush => true #boolean
- #source => string
- stream_identity => "%{host}.%{file}.%{type}" # string
- what => previous
- }
- grok {
- match => {
- "message" => "%{CSIM_LOGLVL:log_level} %{CSIM_TIMESTAMP:[@metadata][timestamp]} %{GREEDYDATA:body}"
- }
- }
- date {
- timezone => "Asia/Tokyo"
- match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
- }
- }
- }
- # cat 103-filter.conf
- filter {
- # type3 - request singleline:
- #Format:
- #Timestamp Body;
- #Body: ClientIP "Verb Request HTTP/1.1" Response Bytes
- if [log_file] in ["ad-request","clientweb-request","crmserver-request","portal-request"] {
- mutate {
- add_field => {
- "format" => "103 - request"
- "received_at" => "%{@timestamp}"
- }
- }
- grok {
- named_captures_only => true
- match => {
- "message" => "%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>%{IP:clientip} \"%{WORD:verb} %{URIPATHPARAM:request} HTTP\/(?<http_version>\d\.\d)\" %{NUMBER:response:int} %{NUMBER:bytes:int})"
- }
- }
- date {
- #timezone => "America/Los_Angeles"
- #timezone => "Europe/Kiev"
- timezone => "Asia/Tokyo"
- match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
- }
- }
- }
- # 104-filter.conf
- filter {
- # type 4.1 - scanner - attention there are multiline:
- #Format:
- # Loglvl Timestamp Body;
- # Body: [logger] who - what
- if [log_file] in ["ad-server","clientweb-server","crm-server","portal-server","ad-client","portal-client","aggregator","cfgsrv","dbsrv","emailserver","scheduler"] {
- mutate {
- add_field => {
- "format" => "104 - server"
- "received_at" => "%{@timestamp}"
- }
- }
- if [log_file] not in ["clientweb-server","crm-server","aggregator","dbsrv"] {
- multiline {
- #allow_duplicates => boolean
- max_age => 60
- negate => true
- pattern => "^%{CSIM_LOGLVL}\s+%{CSIM_TIMESTAMP}"
- #patterns_dir => array
- periodic_flush => true #boolean
- #source => string
- stream_identity => "%{host}.%{file}.%{type}" # string
- what => previous
- }
- }
- grok {
- match => {
- #"message" => "%{CSIM_LOGLVL:log_level}\s+%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>\[(?<logger>.+)\] (?<who>.+?)\s+-\s+(?<what>.*))"
- "message" => "%{CSIM_LOGLVL:log_level}\s+%{CSIM_TIMESTAMP:[@metadata][timestamp]} (?<body>.*)"
- }
- }
- date {
- #timezone => "America/Los_Angeles"
- #timezone => "Europe/Kiev"
- timezone => "Asia/Tokyo"
- match => [ "[@metadata][timestamp]" , "yyyy/MM/dd HH:mm:ss.SSS" ]
- }
- }
- }
- # 105-filter.conf
- filter {
- if [log_file] in ["ad.wrapper","clientweb.wrapper","crmserver.wrapper","portal.wrapper"] {
- mutate {
- add_field => {
- "format" => "105 - wrapper"
- "received_at" => "%{@timestamp}"
- }
- }
- grok {
- break_on_match => false
- match => {
- "message" => "%{CSIM_LOGLVL:log_level}\|(?<logger>.+?)\|%{TIMESTAMP_ISO8601:[@metadata][timestamp]}\|%{GREEDYDATA:body}"
- }
- }
- date {
- #timezone => "America/Los_Angeles"
- #timezone => "Europe/Kiev"
- timezone => "Asia/Tokyo"
- match => [ "[@metadata][timestamp]" , "yy-MM-dd HH:mm:ss" ]
- }
- }
- }
- # cat 107-test_label-filter.conf
- filter {
- mutate {
- add_field => { "test_label" => "stress_test_dialer" }
- }
- }
- # cat 202-elasticsearch-output.conf
- output {
- if [type] != "load-test" {
- elasticsearch {
- host => "localhost"
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement