Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- type = "http"
- method = "post"
- uri = "https://in.logs.betterstack.com/metrics"
- encoding.codec = "json"
- auth.strategy = "bearer"
- auth.token = "enfUhPmsJT6tbkBC5jpkxe9L"
- inputs = ["better_stack_docker_metrics_parser_enfUhPmsJT6tbkBC5jpkxe9L"]
- # --- end of 2023-08-18: https://logs.betterstack.com/vector-toml/ubuntu/enfUhPmsJT6tbkBC5jpkxe9L
- root@ethernity:/etc/vector# grep vector.toml
- ^C
- root@ethernity:/etc/vector# ^C
- root@ethernity:/etc/vector# ^C
- root@ethernity:/etc/vector# cat vector.yaml
- # __ __ __
- # \ \ / / / /
- # \ V / / /
- # \_/ \/
- #
- # V E C T O R
- # Configuration
- #
- # ------------------------------------------------------------------------------
- # Website: https://vector.dev
- # Docs: https://vector.dev/docs
- # Chat: https://chat.vector.dev
- # ------------------------------------------------------------------------------
- # Change this to use a non-default directory for Vector data storage:
- # data_dir = "/var/lib/vector"
- # Random Syslog-formatted logs
- sources:
- dummy_logs:
- type: "demo_logs"
- format: "syslog"
- interval: 1
- # Parse Syslog logs
- # See the Vector Remap Language reference for more info: https://vrl.dev
- transforms:
- parse_logs:
- type: "remap"
- inputs: ["dummy_logs"]
- source: |
- . = parse_syslog!(string!(.message))
- # Print parsed logs to stdout
- sinks:
- print:
- type: "console"
- inputs: ["parse_logs"]
- encoding:
- codec: "json"
- # Vector's GraphQL API (disabled by default)
- # Uncomment to try it out with the `vector top` command or
- # in your browser at http://localhost:8686
- # api:
- # enabled: true
- # address: "127.0.0.1:8686"
- root@ethernity:/etc/vector#
- root@ethernity:/etc/vector# cat vector.toml
- # __ __ __
- # \ \ / / / /
- # \ V / / /
- # \_/ \/
- #
- # V E C T O R
- # Configuration
- #
- # ------------------------------------------------------------------------------
- # Website: https://vector.dev
- # Docs: https://vector.dev/docs
- # Chat: https://chat.vector.dev
- # ------------------------------------------------------------------------------
- # Change this to use a non-default directory for Vector data storage:
- # data_dir = "/var/lib/vector"
- # Random Syslog-formatted logs
- [sources.dummy_logs]
- type = "demo_logs"
- format = "syslog"
- interval = 1
- # Parse Syslog logs
- # See the Vector Remap Language reference for more info: https://vrl.dev
- [transforms.parse_logs]
- type = "remap"
- inputs = ["dummy_logs"]
- source = '''
- . = parse_syslog!(string!(.message))
- '''
- # Print parsed logs to stdout
- [sinks.print]
- type = "console"
- inputs = ["parse_logs"]
- encoding.codec = "json"
- # Vector's GraphQL API (disabled by default)
- # Uncomment to try it out with the `vector top` command or
- # in your browser at http://localhost:8686
- #[api]
- #enabled = true
- #address = "127.0.0.1:8686"
- # Forwarding logs to Logs.betterstack.com
- # ---------------------------------------
- # Generated on 2023-08-18: https://logs.betterstack.com/vector-toml/ubuntu/enfUhPmsJT6tbkBC5jpkxe9L
- # Learn more about Vector configuration: https://vector.dev/docs/reference/configuration/
- # - Apache2: v5
- # - Nginx: v5
- # - PostgreSQL: v5
- # - MySQL: v5
- # - MongoDB: v5
- # - Redis: v5
- # - Auth log: v5
- # - Uncomplicated Firewall (UFW): v5
- # - Docker: v3
- [sources.better_stack_apache2_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/apache2/*.log"]
- exclude = []
- [transforms.better_stack_apache2_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_apache2_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .apache2 = parse_apache_log(.message, format: "combined") ??
- parse_apache_log(.message, format: "common") ??
- parse_apache_log(.message, format: "error") ??
- {}
- if .apache2 != {} {
- .platform = "Apache2"
- .dt = format_timestamp!(parse_timestamp(del(.apache2.timestamp), "%d/%b/%Y:%T %z") ?? .dt, "%+")
- .level = del(.apache2.severity)
- .message = del(.apache2.message)
- request_msg = (string(.apache2.method) ?? "") + " " + (string(.apache2.path) ?? "") + " " + (string(.apache2.protocol) ?? "")
- if .message == request_msg { del(.message) }
- } else {
- del(.apache2)
- }
- '''
- [sources.better_stack_nginx_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/nginx/*.log"]
- exclude = []
- [transforms.better_stack_nginx_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_nginx_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .nginx = parse_regex(.message, r'^\s*(-|(?P<client>\S+))\s+\-\s+(-|(?P<user>\S+))\s+\[(?P<timestamp>.+)\]\s+"(?P<request>(?P<method>\w+)\s+(?P<path>\S+)\s+(?P<protocol>\S+))"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(-|(?P<referrer>.+))"\s+"(-|(?P<agent>.+))"\s*') ??
- parse_regex(.message, r'^\s*(?P<timestamp>.+)\s+\[(?P<severity>\w+)\]\s+(?P<pid>\d+)\#(?P<tid>\d+):\s+\*(?P<cid>\d+)\s+(?P<message>.*)(?:,\s+client:\s+(?P<client>[^,z]+))(?:,\s+server:\s+(?P<server>[^,z]+))(?:,\s+request:\s+"(?P<request>[^"]+)")(?:,\s+subrequest:\s+"(?P<subrequest>[^"]+)")?(?:,\s+upstream:\s+"(?P<upstream>[^"]+)")?(?:,\s+host:\s+"(?P<host>[^"]+)")(?:,\s+referrer:\s+"(?P<referrer>[^"]+)")?\s*') ??
- parse_nginx_log(.message, format: "combined") ??
- parse_nginx_log(.message, format: "error") ??
- {}
- if .nginx != {} {
- .platform = "Nginx"
- .level = del(.nginx.severity)
- .message = del(.nginx.message)
- if is_null(.message) { del(.message) }
- if exists(.nginx.timestamp) {
- .dt = format_timestamp!(
- parse_timestamp(.nginx.timestamp, "%d/%b/%Y:%T %z") ??
- parse_timestamp(.nginx.timestamp, "%Y/%m/%d %T") ??
- .dt,
- "%+"
- )
- del(.nginx.timestamp)
- }
- if is_string(.nginx.status) { .nginx.status = to_int(.nginx.status) ?? .nginx.status }
- if is_string(.nginx.size) { .nginx.size = to_int(.nginx.size) ?? .nginx.size }
- if is_string(.nginx.cid) { .nginx.cid = to_int(.nginx.cid) ?? .nginx.cid }
- if is_string(.nginx.pid) { .nginx.pid = to_int(.nginx.pid) ?? .nginx.pid }
- if is_string(.nginx.tid) { .nginx.tid = to_int(.nginx.tid) ?? .nginx.tid }
- if is_null(.nginx.subrequest) { del(.nginx.subrequest) }
- if is_null(.nginx.upstream) { del(.nginx.upstream) }
- if is_null(.nginx.referrer) { del(.nginx.referrer) }
- } else {
- del(.nginx)
- }
- '''
- [sources.better_stack_postgresql_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/postgresql/*.log"]
- exclude = []
- [transforms.better_stack_postgresql_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_postgresql_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .postgres = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+ \d+:\d+:\d+\.\d+ \S+) \[(?P<pid>\d+)\] (?:(?:(?P<username>\S+)@(?P<database>\S+))?\s*(?P<level>\w+):\s*(?P<message>.*))?') ?? {}
- if .postgres != {} {
- .platform = "PostgreSQL"
- if exists(.postgres.pid) { .postgres.pid = to_int!(.postgres.pid) }
- if exists(.postgres.level) { .level = downcase!(del(.postgres.level)) }
- if exists(.postgres.message) { .message = del(.postgres.message) }
- if exists(.postgres.dt) {
- parsed_dt = to_string(.postgres.dt)
- if !ends_with(parsed_dt, " UTC") {
- .postgres.local_date_time = del(.postgres.dt)
- } else {
- .dt = del(.postgres.dt)
- }
- }
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.postgres)
- }
- '''
- [sources.better_stack_mysql_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/mysql/*.log"]
- exclude = []
- [transforms.better_stack_mysql_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_mysql_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .mysql = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s+(?P<thread>\d+)\s+\[(?P<priority>\S+)\]\s*\[(?P<error_code>\S+)\]\s*\[(?P<subsystem>\S+)\]\s*(?P<message>.*)') ??
- parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s*(?P<id>\d+)\s*(?P<command>[^\t]+)\s*(?P<message>.*)?') ??
- {}
- if .mysql != {} {
- .platform = "MySQL"
- if exists(.mysql.dt) { .dt = del(.mysql.dt) }
- if exists(.mysql.thread) { .mysql.thread = to_int!(.mysql.thread) }
- if exists(.mysql.id) { .mysql.id = to_int!(.mysql.id) }
- .message = del(.mysql.message)
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.mysql)
- }
- '''
- [sources.better_stack_mongodb_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/mongodb/*.log"]
- exclude = []
- [transforms.better_stack_mongodb_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_mongodb_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .mongodb = object!(parse_json(.message) ?? {})
- if .mongodb != {} && exists(.mongodb.t."$$date") {
- .platform = "MongoDB"
- .dt = .mongodb.t."$$date"
- .message = del(.mongodb.msg)
- if .mongodb.s == "I" {
- .level = "info"
- } else if .mongodb.s == "W" {
- .level = "warning"
- } else if .mongodb.s == "E" {
- .level = "error"
- } else if .mongodb.s == "F" {
- .level = "fatal"
- } else if .mongodb.s == "D" {
- .level = "debug"
- } else if .mongodb.s == "D1" {
- .level = "debug1"
- } else if .mongodb.s == "D2" {
- .level = "debug2"
- } else if .mongodb.s == "D3" {
- .level = "debug3"
- } else if .mongodb.s == "D4" {
- .level = "debug4"
- } else if .mongodb.s == "D5" {
- .level = "debug5"
- }
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.mongodb)
- }
- '''
- [sources.better_stack_redis_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/redis/*.log"]
- exclude = []
- [transforms.better_stack_redis_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_redis_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .redis = parse_regex(.message, r'^(?P<pid>\d+):(?P<role_char>\w)\s+(?P<dt>\d+ \w+ \d+ \d{2}:\d{2}:\d{2}.\d+) (?P<level_symbol>.) (?P<message>.*)') ??
- parse_regex(.message, r'^(?P<pid>\d+):(?P<role>\S+) \((?P<unix_timestamp>\d+)\) (?P<message>.*)') ??
- {}
- if .redis != {} {
- .platform = "Redis"
- if .redis.role_char == "M" {
- .redis.role = "master"
- } else if .redis.role_char == "S" {
- .redis.role = "slave"
- } else if .redis.role_char == "C" {
- .redis.role = "RDB/AOF writing child"
- } else if .redis.role_char == "X" {
- .redis.role = "sentinel"
- }
- if .redis.level_symbol == "." {
- .level = "debug"
- } else if .redis.level_symbol == "-" {
- .level = "verbose"
- } else if .redis.level_symbol == "*" {
- .level = "notice"
- } else if .redis.level_symbol == "#" {
- .level = "warning"
- }
- .redis.pid = to_int(.redis.pid) ?? null
- if exists(.redis.dt) {
- .dt = format_timestamp!(parse_timestamp(del(.redis.dt), "%d %b %Y %T%.f") ?? now(), "%+")
- } else if exists(.redis.unix_timestamp) {
- .dt = format_timestamp!(to_timestamp(del(.redis.unix_timestamp)) ?? now(), "%+")
- }
- .message = del(.redis.message)
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.redis)
- }
- '''
- [sources.better_stack_auth_log_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/auth.log"]
- exclude = []
- [transforms.better_stack_auth_log_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_auth_log_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .auth_log = parse_linux_authorization(.message) ?? {}
- if .auth_log != {} {
- .platform = "Linux Authorization"
- .dt = del(.auth_log.timestamp)
- .message = del(.auth_log.message)
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.auth_log)
- }
- '''
- [sources.better_stack_ufw_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = ["/var/log/ufw.log"]
- exclude = []
- [transforms.better_stack_ufw_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_ufw_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .ufw = parse_regex(.message, r'(?P<local_date_time>\w+\s+\d+\s+\d{2}:\d{2}:\d{2})\s+(?P<host>\S+)\s+kernel:\s*(?:\[\s*(?P<kernel_time>\d+(?:.\d+))?\s*\])?\s*\[UFW (?P<action>\w+)\]\s+(?P<data>.*)') ?? {}
- if .ufw != {} {
- .platform = "UFW"
- if exists(.ufw.kernel_time) { .ufw.kernel_time = to_float!(.ufw.kernel_time) }
- .ufw |= parse_regex(.ufw.data, r'\bIN=(?P<in>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bOUT=(?P<out>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bSRC=(?P<src>(?:\d{1,3}\.\b){3}\d{1,3})(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bDST=(?P<dst>(?:\d{1,3}\.\b){3}\d{1,3})(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bPROTO=(?P<proto>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bMAC=(?P<mac>(?:\w{2}:)*\w{2})(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bTOS=(?P<tos>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bPREC=(?P<prec>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bRES=(?P<res>\S+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bTTL=(?P<ttl>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bWINDOW=(?P<window>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bDPT=(?P<dpt>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bID=(?P<id>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bLEN=(?P<len>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bSPT=(?P<spt>\d+)(?:$|\s)') ?? {}
- .ufw |= parse_regex(.ufw.data, r'\bURGP=(?P<urgp>\d+)(?:$|\s)') ?? {}
- .ufw.ttl = to_int(.ufw.ttl) ?? .ufw.ttl
- .ufw.window = to_int(.ufw.window) ?? .ufw.window
- .ufw.dpt = to_int(.ufw.dpt) ?? .ufw.dpt
- .ufw.id = to_int(.ufw.id) ?? .ufw.id
- .ufw.len = to_int(.ufw.len) ?? .ufw.len
- .ufw.spt = to_int(.ufw.spt) ?? .ufw.spt
- .ufw.urgp = to_int(.ufw.urgp) ?? .ufw.urgp
- ufw_data = string!(.ufw.data)
- if match(ufw_data, r'\bSYN(?:$|\s)') { .ufw.syn = true }
- if match(ufw_data, r'\bDF(?:$|\s)') { .ufw.df = true }
- if match(ufw_data, r'\bCWR(?:$|\s)') { .ufw.cwr = true }
- if match(ufw_data, r'\bECE(?:$|\s)') { .ufw.ece = true }
- if match(ufw_data, r'\bACK(?:$|\s)') { .ufw.ack = true }
- if match(ufw_data, r'\bPSH(?:$|\s)') { .ufw.psh = true }
- .message = (.ufw.action + " " + del(.ufw.data)) ?? .message
- } else {
- del(.ufw)
- }
- '''
- [sources.better_stack_docker_logs_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "docker_logs"
- [sources.better_stack_docker_metrics_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "exec"
- command = [ "docker", "stats", "--format", "json" ]
- decoding.codec = "bytes"
- mode = "streaming"
- [transforms.better_stack_docker_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_docker_logs_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- .docker = del(.)
- .dt = del(.docker.dt)
- .message = del(.docker.message)
- .platform = "Docker"
- .apache2 = parse_apache_log(.message, format: "combined") ??
- parse_apache_log(.message, format: "common") ??
- parse_apache_log(.message, format: "error") ??
- {}
- if .apache2 != {} {
- .platform = "Apache2"
- .dt = format_timestamp!(parse_timestamp(del(.apache2.timestamp), "%d/%b/%Y:%T %z") ?? .dt, "%+")
- .level = del(.apache2.severity)
- .message = del(.apache2.message)
- request_msg = (string(.apache2.method) ?? "") + " " + (string(.apache2.path) ?? "") + " " + (string(.apache2.protocol) ?? "")
- if .message == request_msg { del(.message) }
- } else {
- del(.apache2)
- }
- .nginx = parse_regex(.message, r'^\s*(-|(?P<client>\S+))\s+\-\s+(-|(?P<user>\S+))\s+\[(?P<timestamp>.+)\]\s+"(?P<request>(?P<method>\w+)\s+(?P<path>\S+)\s+(?P<protocol>\S+))"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(-|(?P<referrer>.+))"\s+"(-|(?P<agent>.+))"\s*') ??
- parse_regex(.message, r'^\s*(?P<timestamp>.+)\s+\[(?P<severity>\w+)\]\s+(?P<pid>\d+)\#(?P<tid>\d+):\s+\*(?P<cid>\d+)\s+(?P<message>.*)(?:,\s+client:\s+(?P<client>[^,z]+))(?:,\s+server:\s+(?P<server>[^,z]+))(?:,\s+request:\s+"(?P<request>[^"]+)")(?:,\s+subrequest:\s+"(?P<subrequest>[^"]+)")?(?:,\s+upstream:\s+"(?P<upstream>[^"]+)")?(?:,\s+host:\s+"(?P<host>[^"]+)")(?:,\s+referrer:\s+"(?P<referrer>[^"]+)")?\s*') ??
- parse_nginx_log(.message, format: "combined") ??
- parse_nginx_log(.message, format: "error") ??
- {}
- if .nginx != {} {
- .platform = "Nginx"
- .level = del(.nginx.severity)
- .message = del(.nginx.message)
- if is_null(.message) { del(.message) }
- if exists(.nginx.timestamp) {
- .dt = format_timestamp!(
- parse_timestamp(.nginx.timestamp, "%d/%b/%Y:%T %z") ??
- parse_timestamp(.nginx.timestamp, "%Y/%m/%d %T") ??
- .dt,
- "%+"
- )
- del(.nginx.timestamp)
- }
- if is_string(.nginx.status) { .nginx.status = to_int(.nginx.status) ?? .nginx.status }
- if is_string(.nginx.size) { .nginx.size = to_int(.nginx.size) ?? .nginx.size }
- if is_string(.nginx.cid) { .nginx.cid = to_int(.nginx.cid) ?? .nginx.cid }
- if is_string(.nginx.pid) { .nginx.pid = to_int(.nginx.pid) ?? .nginx.pid }
- if is_string(.nginx.tid) { .nginx.tid = to_int(.nginx.tid) ?? .nginx.tid }
- if is_null(.nginx.subrequest) { del(.nginx.subrequest) }
- if is_null(.nginx.upstream) { del(.nginx.upstream) }
- if is_null(.nginx.referrer) { del(.nginx.referrer) }
- } else {
- del(.nginx)
- }
- # we can't distinguish apache and nginx logs (they have the same format)
- # so we merge them under the generalized "HTTP" platform
- if .platform == "Apache2" || .platform == "Nginx" {
- apache2 = del(.apache2)
- if is_null(apache2) { apache2 = {} }
- nginx = del(.nginx)
- if is_null(nginx) { nginx = {} }
- .platform = "HTTP"
- .http = merge!(apache2, nginx)
- }
- .postgres = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+ \d+:\d+:\d+\.\d+ \S+) \[(?P<pid>\d+)\] (?:(?:(?P<username>\S+)@(?P<database>\S+))?\s*(?P<level>\w+):\s*(?P<message>.*))?') ?? {}
- if .postgres != {} {
- .platform = "PostgreSQL"
- if exists(.postgres.pid) { .postgres.pid = to_int!(.postgres.pid) }
- if exists(.postgres.level) { .level = downcase!(del(.postgres.level)) }
- if exists(.postgres.message) { .message = del(.postgres.message) }
- if exists(.postgres.dt) {
- parsed_dt = to_string(.postgres.dt)
- if !ends_with(parsed_dt, " UTC") {
- .postgres.local_date_time = del(.postgres.dt)
- } else {
- .dt = del(.postgres.dt)
- }
- }
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.postgres)
- }
- .mysql = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s+(?P<thread>\d+)\s+\[(?P<priority>\S+)\]\s*\[(?P<error_code>\S+)\]\s*\[(?P<subsystem>\S+)\]\s*(?P<message>.*)') ??
- parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s*(?P<id>\d+)\s*(?P<command>[^\t]+)\s*(?P<message>.*)?') ??
- {}
- if .mysql != {} {
- .platform = "MySQL"
- if exists(.mysql.dt) { .dt = del(.mysql.dt) }
- if exists(.mysql.thread) { .mysql.thread = to_int!(.mysql.thread) }
- if exists(.mysql.id) { .mysql.id = to_int!(.mysql.id) }
- .message = del(.mysql.message)
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.mysql)
- }
- .mongodb = object!(parse_json(.message) ?? {})
- if .mongodb != {} && exists(.mongodb.t."$$date") {
- .platform = "MongoDB"
- .dt = .mongodb.t."$$date"
- .message = del(.mongodb.msg)
- if .mongodb.s == "I" {
- .level = "info"
- } else if .mongodb.s == "W" {
- .level = "warning"
- } else if .mongodb.s == "E" {
- .level = "error"
- } else if .mongodb.s == "F" {
- .level = "fatal"
- } else if .mongodb.s == "D" {
- .level = "debug"
- } else if .mongodb.s == "D1" {
- .level = "debug1"
- } else if .mongodb.s == "D2" {
- .level = "debug2"
- } else if .mongodb.s == "D3" {
- .level = "debug3"
- } else if .mongodb.s == "D4" {
- .level = "debug4"
- } else if .mongodb.s == "D5" {
- .level = "debug5"
- }
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.mongodb)
- }
- .redis = parse_regex(.message, r'^(?P<pid>\d+):(?P<role_char>\w)\s+(?P<dt>\d+ \w+ \d+ \d{2}:\d{2}:\d{2}.\d+) (?P<level_symbol>.) (?P<message>.*)') ??
- parse_regex(.message, r'^(?P<pid>\d+):(?P<role>\S+) \((?P<unix_timestamp>\d+)\) (?P<message>.*)') ??
- {}
- if .redis != {} {
- .platform = "Redis"
- if .redis.role_char == "M" {
- .redis.role = "master"
- } else if .redis.role_char == "S" {
- .redis.role = "slave"
- } else if .redis.role_char == "C" {
- .redis.role = "RDB/AOF writing child"
- } else if .redis.role_char == "X" {
- .redis.role = "sentinel"
- }
- if .redis.level_symbol == "." {
- .level = "debug"
- } else if .redis.level_symbol == "-" {
- .level = "verbose"
- } else if .redis.level_symbol == "*" {
- .level = "notice"
- } else if .redis.level_symbol == "#" {
- .level = "warning"
- }
- .redis.pid = to_int(.redis.pid) ?? null
- if exists(.redis.dt) {
- .dt = format_timestamp!(parse_timestamp(del(.redis.dt), "%d %b %Y %T%.f") ?? now(), "%+")
- } else if exists(.redis.unix_timestamp) {
- .dt = format_timestamp!(to_timestamp(del(.redis.unix_timestamp)) ?? now(), "%+")
- }
- .message = del(.redis.message)
- # extract message metadata
- tmp = string!(.message)
- .message_metadata = {}
- ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b') if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
- if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
- if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
- if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
- if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
- # we match only full IPv6 addresses
- ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
- if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
- if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
- if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
- if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
- if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
- numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
- if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
- if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
- if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
- if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
- if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
- if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
- if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
- if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
- if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
- if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
- } else {
- del(.redis)
- }
- '''
- [transforms.better_stack_docker_metrics_parser_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "remap"
- inputs = ["better_stack_docker_metrics_enfUhPmsJT6tbkBC5jpkxe9L"]
- source = '''
- del(.source_type)
- .dt = del(.timestamp)
- raw_data = .message
- dt = .timestamp
- host = .host
- pid = .pid
- . = []
- data, err = parse_json(strip_ansi_escape_codes(string!(raw_data)))
- if err == null {
- container_id = data.Container
- container_name = data.Name
- gauges = {
- "cpu_percentage": to_float(replace(data.CPUPerc, "%", "") ?? "") ?? null,
- "memory_percentage": to_float(replace(data.MemPerc, "%", "") ?? "") ?? null,
- "memory_used_bytes": split(data.MemUsage, " / ")[0] ?? null,
- "memory_limit_bytes": split(data.MemUsage, " / ")[1] ?? null,
- "block_in_bytes": split(data.BlockIO, " / ")[0] ?? null,
- "block_out_bytes": split(data.BlockIO, " / ")[1] ?? null,
- "network_in_bytes": split(data.NetIO, " / ")[0] ?? null,
- "network_out_bytes": split(data.NetIO, " / ")[1] ?? null,
- "pids_count": to_int(data.PIDs) ?? null
- }
- gauges_bytes = map_values(gauges) -> |data| {
- if !is_string(data) {
- data
- } else {
- data = string!(data)
- if !ends_with(data, "B") {
- data
- } else if ends_with(data, "TiB") {
- round(1024 * 1024 * 1024 * 1024 * to_float(replace(data, "TiB", "")) ?? null) ?? null
- } else if ends_with(data, "GiB") {
- round(1024 * 1024 * 1024 * to_float(replace(data, "GiB", "")) ?? null) ?? null
- } else if ends_with(data, "MiB") {
- round(1024 * 1024 * to_float(replace(data, "MiB", "")) ?? null) ?? null
- } else if ends_with(data, "KiB") {
- round(1024 * to_float(replace(data, "KiB", "")) ?? null) ?? null
- } else if ends_with(data, "TB") {
- round(1000 * 1000 * 1000 * 1000 * to_float(replace(data, "TB", "")) ?? null) ?? null
- } else if ends_with(data, "GB") {
- round(1000 * 1000 * 1000 * to_float(replace(data, "GB", "")) ?? null) ?? null
- } else if ends_with(data, "MB") {
- round(1000 * 1000 * to_float(replace(data, "MB", "")) ?? null) ?? null
- } else if ends_with(data, "kB") {
- round(1000 * to_float(replace(data, "kB", "")) ?? null) ?? null
- } else {
- round(to_int(replace(data, "B", "")) ?? null) ?? null
- }
- }
- }
- for_each(gauges_bytes) -> |name, value| {
- . = push(., {
- "name": name,
- "kind": "absolute",
- "gauge": {
- "value": value
- },
- "tags": {
- "pid": to_string!(pid),
- "host": to_string!(host),
- "container_id": to_string!(container_id),
- "container_name": to_string!(container_name)
- },
- "dt": dt
- })
- }
- }
- '''
- [sources.better_stack_other_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "file"
- read_from = "beginning"
- ignore_older_secs = 600
- include = [
- "/var/log/*.log",
- "/var/log/**/*.log"
- ]
- exclude = [
- "/var/log/apache2/*.log",
- "/var/log/nginx/*.log",
- "/var/log/postgresql/*.log",
- "/var/log/mysql/*.log",
- "/var/log/mongodb/*.log",
- "/var/log/redis/*.log",
- "/var/log/auth.log",
- "/var/log/ufw.log",
- "/var/log/docker/*.log"
- ]
- [sinks.better_stack_http_sink_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "http"
- method = "post"
- uri = "https://in.logs.betterstack.com/"
- encoding.codec = "json"
- auth.strategy = "bearer"
- auth.token = "enfUhPmsJT6tbkBC5jpkxe9L"
- inputs = [
- "better_stack_apache2_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_nginx_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_postgresql_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_mysql_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_mongodb_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_redis_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_auth_log_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_ufw_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_docker_parser_enfUhPmsJT6tbkBC5jpkxe9L",
- "better_stack_other_enfUhPmsJT6tbkBC5jpkxe9L"
- ]
- [sinks.better_stack_http_metrics_sink_enfUhPmsJT6tbkBC5jpkxe9L]
- type = "http"
- method = "post"
- uri = "https://in.logs.betterstack.com/metrics"
- encoding.codec = "json"
- auth.strategy = "bearer"
- auth.token = "enfUhPmsJT6tbkBC5jpkxe9L"
- inputs = ["better_stack_docker_metrics_parser_enfUhPmsJT6tbkBC5jpkxe9L"]
- # --- end of 2023-08-18: https://logs.betterstack.com/vector-toml/ubuntu/enfUhPmsJT6tbkBC5jpkxe9L
Advertisement
Add Comment
Please, Sign In to add comment