{:timestamp=>"2017-02-07T17:42:47.743000+0000", :message=>"fetched an invalid config", :config=>"input {\n\n    # ------------ NOT NEEDED ---------------\n\n    #timing input\n    #file {\n    #        path => \"/logs/eurostar/**/*timing*.log\"\n    #        start_position => \"beginning\"\n    #        #sincedb_path => \"/sincedb/timinglog\"\n    #        type => \"timing\"\n    #        \"add_field\" => { \"client\" => \"eurostar\" }\n    #   }\n\n    # application input\n    #file {\n    #        path => \"/logs/eurostar/**/*media-server*.log\"\n    #        start_position => \"beginning\"\n    #        #sincedb_path => \"/sincedb/applicationlog\"\n    #        #codec => multiline {\n    #        #    # Grok pattern names are valid! :)\n    #        #    pattern => \"^\\[%{LOGLEVEL}\\]\"\n    #        #    negate => true\n    #        #    what => previous\n    #       #}\n    #        type => \"application\"\n    #        \"add_field\" => { \"client\" => \"eurostar\" }\n    #}\n\n    #timing input\n    #file {\n    #        path => \"/logs/virgin_westcoast/**/*timing*.log\"\n    #        start_position => \"beginning\"\n    #        #sincedb_path => \"/sincedb/timinglog\"\n    #        type => \"timing\"\n    #        \"add_field\" => { \"client\" => \"virgin_westcoast\" }\n    #   }\n\n    # VIRGIN section\n    #file {\n    #        path => \"/logs/virgin_westcoast/**/*media-server*.log\"\n    #        start_position => \"beginning\"\n    #        #sincedb_path => \"/sincedb/applicationlog\"\n    #        #codec => multiline {\n    #        #    # Grok pattern names are valid! :)\n    #        #    pattern => \"^\\[%{LOGLEVEL}\\]\"\n    #        #    negate => true\n    #        #    what => previous\n    #        #}\n    #        type => \"application\"\n    #        \"add_field\" => { \"client\" => \"virgin_westcoast\" }\n    #}\n\n    # ------------ NOT NEEDED ---------------\n\n    #Sync log input EUROSTAR\n    file {\n            path => \"/opt/logstash/input/eurostar/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_eurostar_synclog\"\n            \"add_field\" => { \"client\" => \"eurostar\" }\n    }\n\n    # Events log input EUROSTAR\n    file {\n            path => \"/opt/logstash/input/eurostar/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_eurostar_eventlog\"\n            \"add_field\" => { \"client\" => \"eurostar\" }\n#            max_open_files => 3500\n            close_older => 5\n    }\n\n    #Sync log input VIRGIN_WESTCOAST\n    file {\n            path => \"/opt/logstash/input/virgin_westcoast/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_virgin_westcoast_synclog\"\n            \"add_field\" => { \"client\" => \"virgin_westcoast\" }\n    }\n\n    # Events log input VIRGIN_WESTCOAST\n    file {\n            path => \"/opt/logstash/input/virgin_westcoast/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_virgin_westcoast_eventlog\"\n            \"add_field\" => { \"client\" => \"virgin_westcoast\" }\n#            max_open_files => 3500\n            close_older => 5\n       }\n\n    # Sync log input VIRGIN_EASTCOAST\n    file {\n            path => \"/opt/logstash/input/virgin_eastcoast/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_virgin_eastcoast_synclog\"\n            \"add_field\" => { \"client\" => \"virgin_eastcoast\" }\n       }\n\n    # Events log input VIRGIN_EASTCOAST\n    file {\n            path => \"/opt/logstash/input/virgin_eastcoast/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            # set to remove some test logs from taht date\n            exclude => \"*2016-05-16*\"\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_virgin_eastcoast_eventlog\"\n            \"add_field\" => { \"client\" => \"virgin_eastcoast\" }\n#            max_open_files => 3500\n            close_older => 5\n      }\n    \n     # Sync log input NATIONAL_EXPRESS\n     file {\n            path => \"/opt/logstash/input/national_express/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_national_express_synclog\"\n            \"add_field\" => { \"client\" => \"national_express\" }\n       }\n\n     # Events log input NATIONAL_EXPRESS\n     file {\n            path => \"/opt/logstash/input/national_express/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_national_express_eventlog\"\n            \"add_field\" => { \"client\" => \"national_express\" }\n#            max_open_files => 3500\n            close_older => 5\n      }\n\n     # Sync log input NATIONAL_EXPRESS_BUS\n     file {\n            path => \"/opt/logstash/input/national_express_bus/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_national_express_bus_synclog\"\n            \"add_field\" => { \"client\" => \"national_express_bus\" }\n       }\n\n     # Events log input NATIONAL_EXPRESS_BUS\n     file {\n            path => \"/opt/logstash/input/national_express_bus/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_national_express_bus_eventlog\"\n            \"add_field\" => { \"client\" => \"national_express_bus\" }\n#            max_open_files => 3500\n            close_older => 5\n      }\n\n     # Sync log input C2C\n     file {\n            path => \"/opt/logstash/input/c2c/**/*syncLog*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"sync\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_c2c_synclog\"\n            \"add_field\" => { \"client\" => \"c2c\" }\n       }\n\n     # Events log input C2C\n     file {\n            path => \"/opt/logstash/input/c2c/**/*event*.log\"\n            start_position => \"beginning\"\n            ignore_older => 0\n            type => \"events\"\n            sincedb_path => \"/opt/logstash/sincedb/sincedb_c2c_eventlog\"\n            \"add_field\" => { \"client\" => \"c2c\" }\n#            max_open_files => 3500\n            close_older => 5\n      }\n\n}\n\nfilter {\n\n    if [type] == \"application\" {\n\n        multiline {\n                pattern => \"^\\[%{LOGLEVEL}\\]\"\n                negate => true\n                what => \"previous\"\n        }\n\n        grok {\n            break_on_match => false\n            patterns_dir => \"/etc/logstash/patterns\"\n            match => {\n                \"message\" => \"\\[%{LOGLEVEL:level}\\] \\[%{IPORHOST:from}\\] %{TIMESTAMP_ISO8601:timestamp} \\[%{DATA:thread}\\] \\[%{NOTSPACE:logger}\\] %{GREEDYDATA:msg}\"\n                \"thread\" => \"(%{GREEDYDATA}%{REQUEST_TYPE:reqType}%{SPACE}%{URIPATH:reqPath}(%{URIPARAM:reqParam})?)?\"\n            }\n            #remove_tag => [\"_grokparsefailure\"]\n            #named_captures_only => false\n        }\n\n       date {\n            locale => \"en\"\n            match => [\"timestamp\", \"ISO8601\"]\n            timezone => \"UTC\"\n            target => \"@timestamp\"\n            add_field => { \"debug\" => \"timestampMatched\"}\n        }\n\n        mutate\n        {\n         remove_field => [ \"timestamp\"]\n        }\n    }\n\n    if [type] == \"sync\" {\n\n        multiline {\n                pattern => \"%{TIMESTAMP_ISO8601}\"\n                negate => true\n                what => \"previous\"\n        }\n\n        grok {\n            break_on_match => false\n            patterns_dir => \"/etc/logstash/patterns\"\n            match => {\n                \"message\" => \"%{TIMESTAMP_ISO8601:timestamp} - %{NOTSPACE:logger} - %{LOGLEVEL:level} - %{NOTSPACE:syncType} - %{NOTSPACE:runid} - %{GREEDYDATA:msg}\"\n                \"msg\" => \"((Number of regular files transferred: %{BIG_NUMBER:filecount})|(Total transferred file size: %{BIG_NUMBER:bytecount}))\"\n            }\n            remove_tag => [\"_grokparsefailure\"]\n            #named_captures_only => true\n        }\n\n        date {\n            locale => \"en\"\n            match => [\"timestamp\", \"YYYY-MM-dd HH:mm:ss,SSS\"]\n            timezone => \"UTC\"\n            target => \"@timestamp\"\n            add_field => { \"debug\" => \"timestampMatched\"}\n        }\n\n        mutate {\n            remove_field => [\"timestamp\"]\n        }\n\n        mutate {\n            gsub => [ \"filecount\", \"\\D\", \"\" ]\n            gsub => [ \"bytecount\", \"\\D\", \"\" ]\n        }\n        mutate {\n            convert => [ \"filecount\", \"integer\" ]\n            convert => [ \"bytecount\", \"integer\" ]\n        }\n    }\n\n    if [type] == \"events\" {\n\n        mutate {\n                gsub => [\"message\", \"[\\u0000]\", \"\"]\n        }\n\n        json {\n            source => \"message\"\n            add_field => { \"file_path\" => \"%{path}\"}\n        }\n\n        if [eventType] == \"WATCH_CONTENT_CLICKED\" and [pageId] =~ \"magazines\" {\n            mutate {\n                update => { \"eventType\" => \"READ_MAGAZINE_CLICKED\" }\n            }\n        }\n\n        if [eventType] == \"CONTENT_CLICKED\" and [pageId] =~ \"magazines\" {\n            mutate {\n                update => { \"eventType\" => \"MAGAZINE_CLICKED\" }\n            }\n        }\n\n\tif [pageId] =~ \"index.html\" {\n            mutate {\n\t\tgsub => ['pageId','index.html',\"\"]\n            }\n        }\n\n        date {\n            locale => \"en\"\n            match => [\"timestamp\", \"ISO8601\"]\n            timezone => \"UTC\"\n            target => \"@timestamp\"\n            add_field => { \"debug\" => \"timestampMatched\"}\n        }\n\n        if [contentId] and [eventType] != \"READ_MAGAZINE_CLICKED\" {\n            translate {\n                field => \"contentId\"\n                destination => \"movieTitle\"\n                exact => true\n                dictionary_path => \"/etc/logstash/mappings/movies.yaml\"\n                fallback => \"No mapping found\"\n            }\n        }\n\n\tif [adId] and [eventType] =~ \"^VAST_*\" {\n            translate {\n                field => \"adId\"\n                destination => \"adTitle\"\n                exact => true\n                dictionary_path => \"/etc/logstash/mappings/ads.yaml\"\n                fallback => \"No mapping found\"\n            }\n        }\n\n        if [from] {\n            translate {\n                field => \"from\"\n                destination => \"trainUnit\"\n                exact => true\n                dictionary_path => \"/etc/logstash/mappings/trains.yaml\"\n                fallback => \"%{from}\"\n            }\n        }\n\n        if [userAgent] {\n            useragent {\n                source => \"userAgent\"\n            }\n        }\n\n\tif [eventType] == \"PAGE_VIEW\" {\n          gomedia_aggregate {\n            task_id => \"%{visitor}\"\n            code => \"\n                  map['page_viewed'] ||= 0;\n                  map['page_viewed'] += 1;\n                  map['int_duration_sum'] ||= 0;\n                  map['userAgent'] ||= event['userAgent'];\n                  map['last_event'] ||= Time.parse(event['timestamp']);\n                  map['file_path'] ||= event['file_path'];\n                  map['session_id'] ||= SecureRandom.hex;\n                  map['client'] ||= event['client'];\n                  event['evt_session_id'] = map['session_id']\n\n                  if map['last_event'] < Time.parse(event['timestamp'])\n                    map['last_event'] = Time.parse(event['timestamp'])\n                  end\"\n\n            flush_on_all_events => true\n\n            timeout_code => \"\n                  event['userAgent'] = map['userAgent'];\n                  event['page_viewed'] = map['page_viewed'];\n                  event['last_event'] = map['last_event'];\n                  event['@timestamp'] = LogStash::Timestamp.new(map['last_event']);\n                  event['int_duration_sum'] = map['int_duration_sum'];\n                  event['eventType'] = 'SESSION';\n                  event['session_id'] = map['session_id'];\n                  event['file_path'] = map['file_path'];\n                  event['type'] = 'events_session';\n                  event['session_duration'] =  map['last_event'] - event['creation_timestamp'];\n                  event['client'] = map['client']\"\n            timeout_id => \"visitor\"\n            timeout => 900\n            periodic_flush => true\n            timestamp_field => \"timestamp\"\n            timestamp_key => \"path\"\n            track_times => true\n          }\n        }\n\n        if [eventType] == \"CONTENT_PLAYBACK\" {\n          gomedia_aggregate {\n            task_id => \"%{visitor}\"\n            code => \"\n                  map['page_viewed'] ||= 0;\n                  map['int_duration_sum'] ||= 0;\n                  map['userAgent'] ||= event['userAgent'];\n                  map['int_duration_sum'] += event['int_duration'];\n                  map['last_event'] ||= Time.parse(event['timestamp']);\n                  map['file_path'] ||= event['file_path'];\n                  map['session_id'] ||= SecureRandom.hex;\n                  map['client'] ||= event['client'];\n\n                  event['evt_session_id'] = map['session_id']\n\n                  if map['last_event'] < Time.parse(event['timestamp'])\n                    map['last_event'] = Time.parse(event['timestamp'])\n                  end\"\n\n            flush_on_all_events => true\n\n            timeout_code => \"\n                    event['userAgent'] = map['userAgent'];\n                    event['page_viewed'] = map['page_viewed'];\n                    event['last_event'] = map['last_event'];\n                    event['@timestamp'] = LogStash::Timestamp.new(map['last_event']);\n                    event['int_duration_sum'] = map['int_duration_sum'];\n                    event['eventType'] = 'SESSION';\n                    event['file_path'] = map['file_path'];\n                    event['session_id'] = map['session_id'];\n                    event['type'] = 'events_session';\n                    event['session_duration'] =  map['last_event'] - event['creation_timestamp'];\n                    event['client'] = map['client']\"\n            timeout_id => \"visitor\"\n            timeout => 900\n            periodic_flush => true\n            timestamp_field => \"timestamp\"\n            timestamp_key => \"path\"\n            track_times => true\n          }\n        }\n\n\tmutate {\n            remove_field => [\"path\", \"timestamp\"]\n        }\n    }\n\n    if [type] == \"timing\" {\n        json {\n            source => \"message\"\n            add_field => { \"file_path\" => \"%{path}\"}\n        }\n\n         date {\n            locale => \"en\"\n            match => [\"timestamp\", \"ISO8601\"]\n            timezone => \"UTC\"\n            target => \"@timestamp\"\n            add_field => { \"debug\" => \"timestampMatched\"}\n        }\n\n\n        mutate {\n            remove_field => [\"timestamp\" ]\n        }\n    }\n}\n\noutput {\n\n    # ---------------- NOT NEEDED ----------------\n    #if [type] == \"timing\" {\n    #\n    #    file {\n    #                path => \"/output/timing.log\"\n    #        }\n    #\n    #    stdout { codec => rubydebug }\n    #\n    #    elasticsearch {\n    #        host => \"127.0.0.1\"\n    #        cluster => \"logstash\"\n    #        index => \"timing-%{client}-%{+YYYY.MM.dd}\"\n    #        template => \"/opt/logstash/templates/timing.json\"\n    #        template_name => \"timing\"\n    #        manage_template => true\n    #    }\n    #}\n\n    #if [type] == \"application\" {\n    #    file {\n    #        path => \"/output/application.log\"\n    #    }\n    #    stdout { codec => rubydebug }\n    #\n    #    elasticsearch {\n    #       host => \"127.0.0.1\"\n    #       cluster => \"logstash\"\n    #\n    #      index => \"application-%{client}-%{+YYYY.MM.dd}\"\n    #       template => \"/opt/logstash/templates/application.json\"\n    #       template_name => \"application\"\n    #       manage_template => true\n    #    }\n    #}\n    # ---------------- NOT NEEDED ----------------\n\n    if [type] == \"sync\" {\n        #stdout { codec => rubydebug }\n        elasticsearch {\n            hosts => [\"localhost:9202\"]\n            index => \"sync-%{client}-%{+YYYY.ww}\"\n            template => \"/opt/logstash/templates/sync.json\"\n            template_name => \"sync\"\n            manage_template => true\n            user => \"reporting_admin\"\n            password => \"Sc8gcqqmNikRP3C4i7KDAVTc\"\n        }\n    }\n\n    if [type] =~ \"^events.*\" {\n        #stdout { codec => rubydebug }\n        if \"_jsonparsefailure\" in [tags] {\n            file {\n                path => \"/opt/logstash/output/events-%{client}-%{+YYYY.ww}_failed_json.log\" \n            }\n        } else {\n            if [eventType] in [\"CONTENT_PLAYBACK\", \"PAGE_VIEW\", \"SESSION\", \"CONTENT_CLICKED\", \"WATCH_CONTENT_CLICKED\", \"AD_IMPRESSION\", \"VAST_IMPRESSION\", \"VAST_START\", \"VAST_FIRSTQUARTILE\", \"VAST_MIDPOINT\", \"VAST_THIRDQUARTILE\", \"VAST_COMPLETE\", \"GAME_CLICKED\", \"MAGAZINE_CLICKED\", \"READ_MAGAZINE_CLICKED\", \"PLAY_NOW_GAME_CLICKED\", \"CONTENT_PLAYBACK_ERROR\", \"LICENSE_UNAVAILABLE_POPUP_SHOWN\", \"SKE_ENTITLEMENT_ERROR\", \"DRM_POPUP_SHOWN\", \"SKE_ENTITLEMENT_SUCCESS\", \"WATCH_TRAILER_CLICKED\"] {\n                 elasticsearch {\n                    hosts => [\"localhost:9202\"]\n                    index => \"events-%{client}-%{+YYYY.ww}\n                    template => \"/opt/logstash/templates/events.json\"\n                    template_name => \"events\"\n                    manage_template => true\n                    user => \"reporting_admin\"\n                    password => \"Sc8gcqqmNikRP3C4i7KDAVTc\"\n                }\n            } else {\n                 elasticsearch {\n                    hosts => [\"localhost:9202\"]\n                    index => \"unknown-events-%{client}-%{+YYYY.ww}\"\n                    template => \"/opt/logstash/templates/events.json\"\n                    template_name => \"events\"\n                    manage_template => true\n                    user => \"reporting_admin\"\n                    password => \"Sc8gcqqmNikRP3C4i7KDAVTc\"\n                }\n            }    \n        }\n    }\n}\n\n\n", :reason=>"Expected one of #, {, } at line 512, column 34 (byte 17499) after output {\n\n    # ---------------- NOT NEEDED ----------------\n    #if [type] == \"timing\" {\n    #\n    #    file {\n    #                path => \"/output/timing.log\"\n    #        }\n    #\n    #    stdout { codec => rubydebug }\n    #\n    #    elasticsearch {\n    #        host => \"127.0.0.1\"\n    #        cluster => \"logstash\"\n    #        index => \"timing-%{client}-%{+YYYY.MM.dd}\"\n    #        template => \"/opt/logstash/templates/timing.json\"\n    #        template_name => \"timing\"\n    #        manage_template => true\n    #    }\n    #}\n\n    #if [type] == \"application\" {\n    #    file {\n    #        path => \"/output/application.log\"\n    #    }\n    #    stdout { codec => rubydebug }\n    #\n    #    elasticsearch {\n    #       host => \"127.0.0.1\"\n    #       cluster => \"logstash\"\n    #\n    #      index => \"application-%{client}-%{+YYYY.MM.dd}\"\n    #       template => \"/opt/logstash/templates/application.json\"\n    #       template_name => \"application\"\n    #       manage_template => true\n    #    }\n    #}\n    # ---------------- NOT NEEDED ----------------\n\n    if [type] == \"sync\" {\n        #stdout { codec => rubydebug }\n        elasticsearch {\n            hosts => [\"localhost:9202\"]\n            index => \"sync-%{client}-%{+YYYY.ww}\"\n            template => \"/opt/logstash/templates/sync.json\"\n            template_name => \"sync\"\n            manage_template => true\n            user => \"reporting_admin\"\n            password => \"Sc8gcqqmNikRP3C4i7KDAVTc\"\n        }\n    }\n\n    if [type] =~ \"^events.*\" {\n        #stdout { codec => rubydebug }\n        if \"_jsonparsefailure\" in [tags] {\n            file {\n                path => \"/opt/logstash/output/events-%{client}-%{+YYYY.ww}_failed_json.log\" \n            }\n        } else {\n            if [eventType] in [\"CONTENT_PLAYBACK\", \"PAGE_VIEW\", \"SESSION\", \"CONTENT_CLICKED\", \"WATCH_CONTENT_CLICKED\", \"AD_IMPRESSION\", \"VAST_IMPRESSION\", \"VAST_START\", \"VAST_FIRSTQUARTILE\", \"VAST_MIDPOINT\", \"VAST_THIRDQUARTILE\", \"VAST_COMPLETE\", \"GAME_CLICKED\", \"MAGAZINE_CLICKED\", \"READ_MAGAZINE_CLICKED\", \"PLAY_NOW_GAME_CLICKED\", \"CONTENT_PLAYBACK_ERROR\", \"LICENSE_UNAVAILABLE_POPUP_SHOWN\", \"SKE_ENTITLEMENT_ERROR\", \"DRM_POPUP_SHOWN\", \"SKE_ENTITLEMENT_SUCCESS\", \"WATCH_TRAILER_CLICKED\"] {\n                 elasticsearch {\n                    hosts => [\"localhost:9202\"]\n                    index => \"events-%{client}-%{+YYYY.ww}\n                    template => \"", :level=>:error}