Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- PUT for the template of the documents:
- PUT /_template/logstash-custom {
- "index_patterns": ["logstash-*"],
- "order": 0,
- "settings": {
- "number_of_shards": 5
- },
- "mappings": {
- "doc": {
- "dynamic_date_formats": ["yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "MM-dd-yyyy HH:mm:ss", "dd/MMM/yyyy:HH:mm:ss Z", "yyyy-MM-dd HH:mm:ss:SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd"],
- "dynamic_templates" : [
- {
- "doc" : {
- "match" : ".*DTM|.*DATE|.*Date",
- "match_pattern": "regex",
- "mapping" : {
- "type" : "date",
- "format" : "MMM dd, yyyy, HH:mm:ss.SSS Z||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd HH:mm:ss||MM-dd-yyyy HH:mm:ss||dd/MMM/yyyy:HH:mm:ss Z||yyyy-MM-dd HH:mm:ss:SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy-MM-dd"
- }
- }
- }
- ],
- "properties": {
- "@timestamp": {
- "type": "date",
- "format": "MMM dd, yyyy, HH:mm:ss.SSS Z||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd HH:mm:ss||MM-dd-yyyy HH:mm:ss||dd/MMM/yyyy:HH:mm:ss Z||yyyy-MM-dd HH:mm:ss:SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy-MM-dd"
- },
- "@version": {
- "type": "keyword"
- },
- "beat": {
- "properties": {
- "hostname": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "name": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "version": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
- "call": {
- "properties": {
- "ACCT_NBR": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "ANI": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "AO": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "APP": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "CPA_RESULT": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "CTI_CALLID": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "DNIS": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "JOB_ID": {
- "type": "keyword",
- "eager_global_ordinals": true,
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "LINE_NBR": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "PEG": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "END_DTM": {
- "type": "date",
- "format": "MM-dd-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd HH:mm:ss||epoch_millis"
- },
- "START_DTM": {
- "type": "date",
- "format": "MM-dd-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd HH:mm:ss||epoch_millis"
- },
- "REQUEST": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "RESPONSE": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "RESPONSE_STATUS": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "SEQ_NBR": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "UNIQUE_JOBID": {
- "type": "keyword",
- "eager_global_ordinals": true,
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "XFER_GROUP": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "XFER_NBR": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "XFER_PEG": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "XFER_STATUS": {
- "type": "keyword",
- "eager_global_ordinals": true,
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
- "client_ip": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "function": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "geoip": {
- "dynamic": "true",
- "properties": {
- "ip": {
- "type": "ip"
- },
- "latitude": {
- "type": "half_float"
- },
- "location": {
- "type": "geo_point"
- },
- "longitude": {
- "type": "half_float"
- }
- }
- },
- "log": {
- "properties": {
- "level": {
- "type": "keyword",
- "eager_global_ordinals": true,
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "message": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "type": {
- "type": "keyword",
- "eager_global_ordinals": true,
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
- "message": {
- "type": "text",
- "norms": false
- },
- "offset": {
- "type": "long"
- },
- "request": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "res_code": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "server": {
- "properties": {
- "client": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "name": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- },
- "source": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "tags": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- },
- "true_timestamp": {
- "type": "text",
- "norms": false,
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- }
- }
- actual error:
- `[2018-08-13T13:45:23,096][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"logstash-2018.08.13", :_type=>"doc", :_routing=>nil}, #<LogStash::Event:0x540e8046>], :response=>{"index"=>{"_index"=>"logstash-2018.08.13", "_type"=>"doc", "_id"=>"__ubNGUBs_yBIBeOWtxx", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"expected map for property [fields] on field [MMM dd, yyyy, HH:mm:ss.SSS Z||yyyy-MM-dd'T'HH:mm:ss.SSSZ||yyyy-MM-dd HH:mm:ss||MM-dd-yyyy HH:mm:ss||dd/MMM/yyyy:HH:mm:ss Z||yyyy-MM-dd HH:mm:ss:SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy-MM-dd] or [fields] but got a class java.lang.String"}}}}`
- sample doc that does work, grabbed directly from Kibana:
- {
- "_index": "logstash-2018.08.13",
- "_type": "doc",
- "_id": "lfubNGUBs_yBIBeOZd0G",
- "_version": 1,
- "_score": null,
- "_source": {
- "log.message": " getStateInfoDB(110,PULSE)",
- "log.level": "DEBUG",
- "log.type": "StateDataAccess",
- "offset": 249376066,
- "message": "DEBUG [2018-08-13 13:45:21,146] [StateDataAccess] - : getStateInfoDB(110,PULSE)",
- "tags": [
- "EP-filebeat",
- "beats_input_codec_plain_applied"
- ],
- "server": {
- "name": "EP-host",
- "client": "asyUvd"
- },
- "@timestamp": "2018-08-13T18:45:21.146Z",
- "@version": "1",
- "source": "/opt/tomcat/logs/catalina.out",
- "beat": {
- "name": "example name",
- "version": "6.3.2",
- "hostname": "hostname.local"
- }
- },
- "fields": {
- "@timestamp": [
- "2018-08-13T18:45:21.146Z"
- ]
- },
- "sort": [
- 1534185921146
- ]
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement