TechieBraj

vector config

Sep 23rd, 2021
492
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. root@techiebraj:~# cat /etc/vector/vector.toml
  2. #                                    __   __  __
  3. #                                    \ \ / / / /
  4. #                                     \ V / / /
  5. #                                      \_/  \/
  6. #
  7. #                                    V E C T O R
  8. #                                   Configuration
  9. #
  10. # ------------------------------------------------------------------------------
  11. # Website: https://vector.dev
  12. # Docs: https://vector.dev/docs
  13. # Chat: https://chat.vector.dev
  14. # ------------------------------------------------------------------------------
  15.  
  16. # Change this to use a non-default directory for Vector data storage:
  17. # data_dir = "/var/lib/vector"
  18.  
  19. # Random Syslog-formatted logs
  20. [sources.dummy_logs]
  21. type = "generator"
  22. format = "syslog"
  23. interval = 1
  24.  
  25. # Parse Syslog logs
  26. # See the Vector Remap Language reference for more info: https://vrl.dev
  27. [transforms.parse_logs]
  28. type = "remap"
  29. inputs = ["dummy_logs"]
  30. source = '''
  31. . = parse_syslog!(string!(.message))
  32. '''
  33.  
  34. # Print parsed logs to stdout
  35. [sinks.print]
  36. type = "console"
  37. inputs = ["parse_logs"]
  38. encoding.codec = "json"
  39.  
  40. # Vector's GraphQL API (disabled by default)
  41. # Uncomment to try it out with the `vector top` command or
  42. # in your browser at http://localhost:8686
  43. #[api]
  44. #enabled = true
  45. #address = "127.0.0.1:8686"
  46.  
  47. # Forwarding logs to Logtail.com
  48. # ------------------------------
  49. # Generated on 2021-09-23: https://logtail.com/vector-toml/ubuntu/LmXXEqcihhTE5Kf67JPqA8Wk
  50. # Learn more about Vector configuration: https://vector.dev/docs/reference/configuration/
  51.  
  52. # - Apache2: v4
  53. # - Nginx: v4
  54. # - PostgreSQL: v4
  55. # - MySQL: v4
  56. # - MongoDB: v4
  57. # - Redis: v4
  58. # - Auth log: v4
  59. # - Uncomplicated Firewall (UFW): v4
  60. # - Docker: v1
  61.  
  62. [sources.logtail_apache2_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  63. type = "file"
  64. read_from = "beginning"
  65. ignore_older_secs = 600
  66. include = ["/var/log/apache2/*.log"]
  67. exclude = []
  68.  
  69. [transforms.logtail_apache2_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  70. type = "remap"
  71. inputs = ["logtail_apache2_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  72. source = '''
  73. del(.source_type)
  74. .dt = del(.timestamp)
  75. .apache2 = parse_apache_log(.message, format: "combined") ??
  76.     parse_apache_log(.message, format: "common") ??
  77.     parse_apache_log(.message, format: "error") ??
  78.     {}
  79.  
  80. if .apache2 != {} {
  81.   .platform = "Apache2"
  82.  
  83.   .dt = format_timestamp!(parse_timestamp(del(.apache2.timestamp), "%d/%b/%Y:%T %z") ?? .dt, "%+")
  84.   .level = del(.apache2.severity)
  85.   .message = del(.apache2.message)
  86.  
  87.   request_msg = (string(.apache2.method) ?? "") + " " + (string(.apache2.path) ?? "") + " " + (string(.apache2.protocol) ?? "")
  88.   if .message == request_msg { del(.message) }
  89. } else {
  90.   del(.apache2)
  91. }
  92. '''
  93.  
  94. [sources.logtail_nginx_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  95. type = "file"
  96. read_from = "beginning"
  97. ignore_older_secs = 600
  98. include = ["/var/log/nginx/*.log"]
  99. exclude = []
  100.  
  101. [transforms.logtail_nginx_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  102. type = "remap"
  103. inputs = ["logtail_nginx_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  104. source = '''
  105. del(.source_type)
  106. .dt = del(.timestamp)
  107. .nginx = parse_regex(.message, r'^\s*(-|(?P<client>\S+))\s+\-\s+(-|(?P<user>\S+))\s+\[(?P<timestamp>.+)\]\s+"(?P<request>(?P<method>\w+)\s+(?P<path>\S+)\s+(?P<protocol>\S+))"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(-|(?P<referrer>.+))"\s+"(-|(?P<agent>.+))"\s*') ??
  108.     parse_regex(.message, r'^\s*(?P<timestamp>.+)\s+\[(?P<severity>\w+)\]\s+(?P<pid>\d+)\#(?P<tid>\d+):\s+\*(?P<cid>\d+)\s+(?P<message>.*)(?:,\s+client:\s+(?P<client>[^,z]+))(?:,\s+server:\s+(?P<server>[^,z]+))(?:,\s+request:\s+"(?P<request>[^"]+)")(?:,\s+subrequest:\s+"(?P<subrequest>[^"]+)")?(?:,\s+upstream:\s+"(?P<upstream>[^"]+)")?(?:,\s+host:\s+"(?P<host>[^"]+)")(?:,\s+referrer:\s+"(?P<referrer>[^"]+)")?\s*') ??
  109.     parse_nginx_log(.message, format: "combined") ??
  110.     parse_nginx_log(.message, format: "error") ??
  111.     {}
  112.  
  113. if .nginx != {} {
  114.   .platform = "Nginx"
  115.   .level = del(.nginx.severity)
  116.   .message = del(.nginx.message)
  117.  
  118.   if is_null(.message) { del(.message) }
  119.   if exists(.nginx.timestamp) {
  120.     .dt = format_timestamp!(
  121.       parse_timestamp(.nginx.timestamp, "%d/%b/%Y:%T %z") ??
  122.         parse_timestamp(.nginx.timestamp, "%Y/%m/%d %T") ??
  123.         .dt,
  124.       "%+"
  125.     )
  126.  
  127.     del(.nginx.timestamp)
  128.   }
  129.  
  130.   if is_string(.nginx.status) { .nginx.status = to_int(.nginx.status) ?? .nginx.status }
  131.   if is_string(.nginx.size) { .nginx.size = to_int(.nginx.size) ?? .nginx.size }
  132.   if is_string(.nginx.cid) { .nginx.cid = to_int(.nginx.cid) ?? .nginx.cid }
  133.   if is_string(.nginx.pid) { .nginx.pid = to_int(.nginx.pid) ?? .nginx.pid }
  134.   if is_string(.nginx.tid) { .nginx.tid = to_int(.nginx.tid) ?? .nginx.tid }
  135.  
  136.   if is_null(.nginx.subrequest) { del(.nginx.subrequest) }
  137.   if is_null(.nginx.upstream) { del(.nginx.upstream) }
  138.   if is_null(.nginx.referrer) { del(.nginx.referrer) }
  139. } else {
  140.   del(.nginx)
  141. }
  142. '''
  143.  
  144. [sources.logtail_postgresql_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  145. type = "file"
  146. read_from = "beginning"
  147. ignore_older_secs = 600
  148. include = ["/var/log/postgresql/*.log"]
  149. exclude = []
  150.  
  151. [transforms.logtail_postgresql_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  152. type = "remap"
  153. inputs = ["logtail_postgresql_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  154. source = '''
  155. del(.source_type)
  156. .dt = del(.timestamp)
  157. .postgres = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+ \d+:\d+:\d+\.\d+ \S+) \[(?P<pid>\d+)\] (?:(?:(?P<username>\S+)@(?P<database>\S+))?\s*(?P<level>\w+):\s*(?P<message>.*))?') ?? {}
  158.  
  159. if .postgres != {} {
  160.   .platform = "PostgreSQL"
  161.   if exists(.postgres.pid) { .postgres.pid = to_int!(.postgres.pid) }
  162.   if exists(.postgres.level) { .level = downcase!(del(.postgres.level)) }
  163.   if exists(.postgres.message) { .message = del(.postgres.message) }
  164.  
  165.   if exists(.postgres.dt) {
  166.     if !ends_with(.postgres.dt, " UTC") {
  167.       .postgres.local_date_time = del(.postgres.dt)
  168.     } else {
  169.       .dt = del(.postgres.dt)
  170.     }
  171.   }
  172.  
  173.   # extract message metadata
  174.   tmp = to_string(.message) ?? ""
  175.  
  176.   ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  177.   if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  178.   if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  179.   if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  180.   if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  181.   if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  182.  
  183.   # we match only full IPv6 addresses
  184.   ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  185.   if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  186.   if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  187.   if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  188.   if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  189.   if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  190.  
  191.   numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  192.   if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  193.   if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  194.   if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  195.   if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  196.   if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  197.   if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  198.   if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  199.   if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  200.   if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  201.   if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  202.  
  203. } else {
  204.   del(.postgres)
  205. }
  206. '''
  207.  
  208. [sources.logtail_mysql_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  209. type = "file"
  210. read_from = "beginning"
  211. ignore_older_secs = 600
  212. include = ["/var/log/mysql/*.log"]
  213. exclude = []
  214.  
  215. [transforms.logtail_mysql_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  216. type = "remap"
  217. inputs = ["logtail_mysql_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  218. source = '''
  219. del(.source_type)
  220. .dt = del(.timestamp)
  221. .mysql = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s+(?P<thread>\d+)\s+\[(?P<priority>\S+)\]\s*\[(?P<error_code>\S+)\]\s*\[(?P<subsystem>\S+)\]\s*(?P<message>.*)') ??
  222.     parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s*(?P<id>\d+)\s*(?P<command>[^\t]+)\s*(?P<message>.*)?') ??
  223.     {}
  224.  
  225. if .mysql != {} {
  226.   .platform = "MySQL"
  227.   if exists(.mysql.dt) { .dt = del(.mysql.dt) }
  228.   if exists(.mysql.thread) { .mysql.thread = to_int!(.mysql.thread) }
  229.   if exists(.mysql.id) { .mysql.id = to_int!(.mysql.id) }
  230.   .message = del(.mysql.message)
  231.  
  232.   # extract message metadata
  233.   tmp = to_string(.message) ?? ""
  234.  
  235.   ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  236.   if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  237.   if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  238.   if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  239.   if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  240.   if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  241.  
  242.   # we match only full IPv6 addresses
  243.   ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  244.   if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  245.   if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  246.   if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  247.   if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  248.   if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  249.  
  250.   numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  251.   if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  252.   if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  253.   if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  254.   if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  255.   if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  256.   if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  257.   if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  258.   if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  259.   if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  260.   if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  261.  
  262. } else {
  263.   del(.mysql)
  264. }
  265. '''
  266.  
  267. [sources.logtail_mongodb_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  268. type = "file"
  269. read_from = "beginning"
  270. ignore_older_secs = 600
  271. include = ["/var/log/mongodb/*.log"]
  272. exclude = []
  273.  
  274. [transforms.logtail_mongodb_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  275. type = "remap"
  276. inputs = ["logtail_mongodb_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  277. source = '''
  278. del(.source_type)
  279. .dt = del(.timestamp)
  280. .mongodb = object!(parse_json(.message) ?? {})
  281.  
  282. if .mongodb != {} && exists(.mongodb.t."$$date") {
  283.   .platform = "MongoDB"
  284.   .dt = .mongodb.t."$$date"
  285.   .message = del(.mongodb.msg)
  286.  
  287.   if .mongodb.s == "I" {
  288.     .level = "info"
  289.   } else if .mongodb.s == "W" {
  290.     .level = "warning"
  291.   } else if .mongodb.s == "E" {
  292.     .level = "error"
  293.   } else if .mongodb.s == "F" {
  294.     .level = "fatal"
  295.   } else if .mongodb.s == "D" {
  296.     .level = "debug"
  297.   } else if .mongodb.s == "D1" {
  298.     .level = "debug1"
  299.   } else if .mongodb.s == "D2" {
  300.     .level = "debug2"
  301.   } else if .mongodb.s == "D3" {
  302.     .level = "debug3"
  303.   } else if .mongodb.s == "D4" {
  304.     .level = "debug4"
  305.   } else if .mongodb.s == "D5" {
  306.     .level = "debug5"
  307.   }
  308.  
  309.   # extract message metadata
  310.   tmp = to_string(.message) ?? ""
  311.  
  312.   ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  313.   if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  314.   if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  315.   if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  316.   if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  317.   if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  318.  
  319.   # we match only full IPv6 addresses
  320.   ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  321.   if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  322.   if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  323.   if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  324.   if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  325.   if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  326.  
  327.   numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  328.   if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  329.   if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  330.   if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  331.   if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  332.   if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  333.   if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  334.   if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  335.   if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  336.   if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  337.   if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  338.  
  339. } else {
  340.   del(.mongodb)
  341. }
  342. '''
  343.  
  344. [sources.logtail_redis_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  345. type = "file"
  346. read_from = "beginning"
  347. ignore_older_secs = 600
  348. include = ["/var/log/redis/*.log"]
  349. exclude = []
  350.  
  351. [transforms.logtail_redis_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  352. type = "remap"
  353. inputs = ["logtail_redis_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  354. source = '''
  355. del(.source_type)
  356. .dt = del(.timestamp)
  357. .redis = parse_regex(.message, r'^(?P<pid>\d+):(?P<role_char>\w)\s+(?P<dt>\d+ \w+ \d+ \d{2}:\d{2}:\d{2}.\d+) (?P<level_symbol>.) (?P<message>.*)') ??
  358.     parse_regex(.message, r'^(?P<pid>\d+):(?P<role>\S+) \((?P<unix_timestamp>\d+)\) (?P<message>.*)') ??
  359.     {}
  360.  
  361. if .redis != {} {
  362.   .platform = "Redis"
  363.   if .redis.role_char == "M" {
  364.     .redis.role = "master"
  365.   } else if .redis.role_char == "S" {
  366.     .redis.role = "slave"
  367.   } else if .redis.role_char == "C" {
  368.     .redis.role = "RDB/AOF writing child"
  369.   } else if .redis.role_char == "X" {
  370.     .redis.role = "sentinel"
  371.   }
  372.  
  373.   if .redis.level_symbol == "." {
  374.     .level = "debug"
  375.   } else if .redis.level_symbol == "-" {
  376.     .level = "verbose"
  377.   } else if .redis.level_symbol == "*" {
  378.     .level = "notice"
  379.   } else if .redis.level_symbol == "#" {
  380.     .level = "warning"
  381.   }
  382.  
  383.   .redis.pid = to_int(.redis.pid) ?? null
  384.  
  385.   if exists(.redis.dt) {
  386.     .dt = format_timestamp!(parse_timestamp(del(.redis.dt), "%d %b %Y %T%.f") ?? now(), "%+")
  387.   } else if exists(.redis.unix_timestamp) {
  388.     .dt = format_timestamp!(to_timestamp(del(.redis.unix_timestamp)) ?? now(), "%+")
  389.   }
  390.  
  391.   .message = del(.redis.message)
  392.  
  393.   # extract message metadata
  394.   tmp = to_string(.message) ?? ""
  395.  
  396.   ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  397.   if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  398.   if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  399.   if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  400.   if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  401.   if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  402.  
  403.   # we match only full IPv6 addresses
  404.   ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  405.   if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  406.   if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  407.   if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  408.   if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  409.   if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  410.  
  411.   numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  412.   if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  413.   if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  414.   if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  415.   if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  416.   if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  417.   if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  418.   if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  419.   if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  420.   if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  421.   if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  422.  
  423. } else {
  424.   del(.redis)
  425. }
  426. '''
  427.  
  428. [sources.logtail_auth_log_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  429. type = "file"
  430. read_from = "beginning"
  431. ignore_older_secs = 600
  432. include = ["/var/log/auth.log"]
  433. exclude = []
  434.  
  435. [transforms.logtail_auth_log_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  436. type = "remap"
  437. inputs = ["logtail_auth_log_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  438. source = '''
  439. del(.source_type)
  440. .dt = del(.timestamp)
  441. .auth_log = parse_linux_authorization(.message) ?? {}
  442.  
  443. if .auth_log != {} {
  444.   .platform = "Linux Authorization"
  445.   .dt = del(.auth_log.timestamp)
  446.   .message = del(.auth_log.message)
  447.  
  448.   # extract message metadata
  449.   tmp = to_string(.message) ?? ""
  450.  
  451.   ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  452.   if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  453.   if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  454.   if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  455.   if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  456.   if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  457.  
  458.   # we match only full IPv6 addresses
  459.   ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  460.   if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  461.   if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  462.   if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  463.   if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  464.   if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  465.  
  466.   numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  467.   if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  468.   if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  469.   if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  470.   if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  471.   if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  472.   if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  473.   if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  474.   if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  475.   if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  476.   if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  477.  
  478. } else {
  479.   del(.auth_log)
  480. }
  481. '''
  482.  
  483. [sources.logtail_ufw_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  484. type = "file"
  485. read_from = "beginning"
  486. ignore_older_secs = 600
  487. include = ["/var/log/ufw.log"]
  488. exclude = []
  489.  
  490. [transforms.logtail_ufw_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  491. type = "remap"
  492. inputs = ["logtail_ufw_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  493. source = '''
  494. del(.source_type)
  495. .dt = del(.timestamp)
  496. .ufw = parse_regex(.message, r'(?P<local_date_time>\w+\s+\d+\s+\d{2}:\d{2}:\d{2})\s+(?P<host>\S+)\s+kernel:\s*(?:\[\s*(?P<kernel_time>\d+(?:.\d+))?\s*\])?\s*\[UFW (?P<action>\w+)\]\s+(?P<data>.*)') ?? {}
  497.  
  498. if .ufw != {} {
  499.   .platform = "UFW"
  500.  
  501.   if exists(.ufw.kernel_time) { .ufw.kernel_time = to_float!(.ufw.kernel_time) }
  502.  
  503.   .ufw |= parse_regex(.ufw.data, r'\bIN=(?P<in>\S+)(?:$|\s)') ?? {}
  504.   .ufw |= parse_regex(.ufw.data, r'\bOUT=(?P<out>\S+)(?:$|\s)') ?? {}
  505.  
  506.   .ufw |= parse_regex(.ufw.data, r'\bSRC=(?P<src>(?:\d{1,3}\.\b){3}\d{1,3})(?:$|\s)') ?? {}
  507.   .ufw |= parse_regex(.ufw.data, r'\bDST=(?P<dst>(?:\d{1,3}\.\b){3}\d{1,3})(?:$|\s)') ?? {}
  508.  
  509.   .ufw |= parse_regex(.ufw.data, r'\bPROTO=(?P<proto>\S+)(?:$|\s)') ?? {}
  510.   .ufw |= parse_regex(.ufw.data, r'\bMAC=(?P<mac>(?:\w{2}:)*\w{2})(?:$|\s)') ?? {}
  511.  
  512.   .ufw |= parse_regex(.ufw.data, r'\bTOS=(?P<tos>\S+)(?:$|\s)') ?? {}
  513.   .ufw |= parse_regex(.ufw.data, r'\bPREC=(?P<prec>\S+)(?:$|\s)') ?? {}
  514.   .ufw |= parse_regex(.ufw.data, r'\bRES=(?P<res>\S+)(?:$|\s)') ?? {}
  515.  
  516.   .ufw |= parse_regex(.ufw.data, r'\bTTL=(?P<ttl>\d+)(?:$|\s)') ?? {}
  517.   .ufw |= parse_regex(.ufw.data, r'\bWINDOW=(?P<window>\d+)(?:$|\s)') ?? {}
  518.   .ufw |= parse_regex(.ufw.data, r'\bDPT=(?P<dpt>\d+)(?:$|\s)') ?? {}
  519.   .ufw |= parse_regex(.ufw.data, r'\bID=(?P<id>\d+)(?:$|\s)') ?? {}
  520.   .ufw |= parse_regex(.ufw.data, r'\bLEN=(?P<len>\d+)(?:$|\s)') ?? {}
  521.   .ufw |= parse_regex(.ufw.data, r'\bSPT=(?P<spt>\d+)(?:$|\s)') ?? {}
  522.   .ufw |= parse_regex(.ufw.data, r'\bURGP=(?P<urgp>\d+)(?:$|\s)') ?? {}
  523.   .ufw.ttl = to_int(.ufw.ttl) ?? .ufw.ttl
  524.   .ufw.window = to_int(.ufw.window) ?? .ufw.window
  525.   .ufw.dpt = to_int(.ufw.dpt) ?? .ufw.dpt
  526.   .ufw.id = to_int(.ufw.id) ?? .ufw.id
  527.   .ufw.len = to_int(.ufw.len) ?? .ufw.len
  528.   .ufw.spt = to_int(.ufw.spt) ?? .ufw.spt
  529.   .ufw.urgp = to_int(.ufw.urgp) ?? .ufw.urgp
  530.  
  531.   if match(.ufw.data, r'\bSYN(?:$|\s)') { .ufw.syn = true }
  532.   if match(.ufw.data, r'\bDF(?:$|\s)') { .ufw.df = true }
  533.   if match(.ufw.data, r'\bCWR(?:$|\s)') { .ufw.cwr = true }
  534.   if match(.ufw.data, r'\bECE(?:$|\s)') { .ufw.ece = true }
  535.   if match(.ufw.data, r'\bACK(?:$|\s)') { .ufw.ack = true }
  536.   if match(.ufw.data, r'\bPSH(?:$|\s)') { .ufw.psh = true }
  537.  
  538.   .message = (.ufw.action + " " + del(.ufw.data)) ?? .message
  539. } else {
  540.   del(.ufw)
  541. }
  542. '''
  543.  
  544. [sources.logtail_docker_logs_LmXXEqcihhTE5Kf67JPqA8Wk]
  545. type = "docker_logs"
  546.  
  547. [transforms.logtail_docker_parser_LmXXEqcihhTE5Kf67JPqA8Wk]
  548. type = "remap"
  549. inputs = ["logtail_docker_logs_LmXXEqcihhTE5Kf67JPqA8Wk"]
  550. source = '''
  551. del(.source_type)
  552. .dt = del(.timestamp)
  553. .docker = del(.)
  554. .dt = del(.docker.dt)
  555. .message = del(.docker.message)
  556. .platform = "Docker"
  557.  
  558. .apache2 = parse_apache_log(.message, format: "combined") ??
  559.     parse_apache_log(.message, format: "common") ??
  560.     parse_apache_log(.message, format: "error") ??
  561.     {}
  562.  
  563. if .apache2 != {} {
  564.   .platform = "Apache2"
  565.  
  566.   .dt = format_timestamp!(parse_timestamp(del(.apache2.timestamp), "%d/%b/%Y:%T %z") ?? .dt, "%+")
  567.   .level = del(.apache2.severity)
  568.   .message = del(.apache2.message)
  569.  
  570.   request_msg = (string(.apache2.method) ?? "") + " " + (string(.apache2.path) ?? "") + " " + (string(.apache2.protocol) ?? "")
  571.   if .message == request_msg { del(.message) }
  572. } else {
  573.   del(.apache2)
  574. }
  575.  
  576. .nginx = parse_regex(.message, r'^\s*(-|(?P<client>\S+))\s+\-\s+(-|(?P<user>\S+))\s+\[(?P<timestamp>.+)\]\s+"(?P<request>(?P<method>\w+)\s+(?P<path>\S+)\s+(?P<protocol>\S+))"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(-|(?P<referrer>.+))"\s+"(-|(?P<agent>.+))"\s*') ??
  577.     parse_regex(.message, r'^\s*(?P<timestamp>.+)\s+\[(?P<severity>\w+)\]\s+(?P<pid>\d+)\#(?P<tid>\d+):\s+\*(?P<cid>\d+)\s+(?P<message>.*)(?:,\s+client:\s+(?P<client>[^,z]+))(?:,\s+server:\s+(?P<server>[^,z]+))(?:,\s+request:\s+"(?P<request>[^"]+)")(?:,\s+subrequest:\s+"(?P<subrequest>[^"]+)")?(?:,\s+upstream:\s+"(?P<upstream>[^"]+)")?(?:,\s+host:\s+"(?P<host>[^"]+)")(?:,\s+referrer:\s+"(?P<referrer>[^"]+)")?\s*') ??
  578.     parse_nginx_log(.message, format: "combined") ??
  579.     parse_nginx_log(.message, format: "error") ??
  580.     {}
  581.  
  582. if .nginx != {} {
  583.   .platform = "Nginx"
  584.   .level = del(.nginx.severity)
  585.   .message = del(.nginx.message)
  586.  
  587.   if is_null(.message) { del(.message) }
  588.   if exists(.nginx.timestamp) {
  589.     .dt = format_timestamp!(
  590.       parse_timestamp(.nginx.timestamp, "%d/%b/%Y:%T %z") ??
  591.         parse_timestamp(.nginx.timestamp, "%Y/%m/%d %T") ??
  592.         .dt,
  593.       "%+"
  594.     )
  595.  
  596.     del(.nginx.timestamp)
  597.   }
  598.  
  599.   if is_string(.nginx.status) { .nginx.status = to_int(.nginx.status) ?? .nginx.status }
  600.   if is_string(.nginx.size) { .nginx.size = to_int(.nginx.size) ?? .nginx.size }
  601.   if is_string(.nginx.cid) { .nginx.cid = to_int(.nginx.cid) ?? .nginx.cid }
  602.   if is_string(.nginx.pid) { .nginx.pid = to_int(.nginx.pid) ?? .nginx.pid }
  603.   if is_string(.nginx.tid) { .nginx.tid = to_int(.nginx.tid) ?? .nginx.tid }
  604.  
  605.   if is_null(.nginx.subrequest) { del(.nginx.subrequest) }
  606.   if is_null(.nginx.upstream) { del(.nginx.upstream) }
  607.   if is_null(.nginx.referrer) { del(.nginx.referrer) }
  608. } else {
  609.   del(.nginx)
  610. }
  611.  
  612. # we can't distinguish apache and nginx logs (they have the same format)
  613. # so we merge them under the generalized "HTTP" platform
  614. if .platform == "Apache2" || .platform == "Nginx" {
  615.  apache2 = del(.apache2)
  616.  if is_null(apache2) { apache2 = {} }
  617.  
  618.  nginx = del(.nginx)
  619.  if is_null(nginx) { nginx = {} }
  620.  
  621.  .platform = "HTTP"
  622.  .http = merge(apache2, nginx)
  623. }
  624.  
  625.  
  626. .postgres = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+ \d+:\d+:\d+\.\d+ \S+) \[(?P<pid>\d+)\] (?:(?:(?P<username>\S+)@(?P<database>\S+))?\s*(?P<level>\w+):\s*(?P<message>.*))?') ?? {}
  627.  
  628. if .postgres != {} {
  629.  .platform = "PostgreSQL"
  630.  if exists(.postgres.pid) { .postgres.pid = to_int!(.postgres.pid) }
  631.  if exists(.postgres.level) { .level = downcase!(del(.postgres.level)) }
  632.  if exists(.postgres.message) { .message = del(.postgres.message) }
  633.  
  634.  if exists(.postgres.dt) {
  635.    if !ends_with(.postgres.dt, " UTC") {
  636.      .postgres.local_date_time = del(.postgres.dt)
  637.    } else {
  638.      .dt = del(.postgres.dt)
  639.    }
  640.  }
  641.  
  642.  # extract message metadata
  643.  tmp = to_string(.message) ?? ""
  644.  
  645.  ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  646.  if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  647.  if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  648.  if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  649.  if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  650.  if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  651.  
  652.  # we match only full IPv6 addresses
  653.  ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  654.  if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  655.  if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  656.  if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  657.  if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  658.  if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  659.  
  660.  numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  661.  if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  662.  if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  663.  if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  664.  if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  665.  if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  666.  if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  667.  if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  668.  if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  669.  if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  670.  if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  671.  
  672. } else {
  673.  del(.postgres)
  674. }
  675.  
  676. .mysql = parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s+(?P<thread>\d+)\s+\[(?P<priority>\S+)\]\s*\[(?P<error_code>\S+)\]\s*\[(?P<subsystem>\S+)\]\s*(?P<message>.*)') ??
  677.    parse_regex(.message, r'^(?P<dt>\d+-\d+-\d+T\d+:\d+:\d+\.\d+\w+)\s*(?P<id>\d+)\s*(?P<command>[^\t]+)\s*(?P<message>.*)?') ??
  678.    {}
  679.  
  680. if .mysql != {} {
  681.  .platform = "MySQL"
  682.  if exists(.mysql.dt) { .dt = del(.mysql.dt) }
  683.  if exists(.mysql.thread) { .mysql.thread = to_int!(.mysql.thread) }
  684.  if exists(.mysql.id) { .mysql.id = to_int!(.mysql.id) }
  685.  .message = del(.mysql.message)
  686.  
  687.  # extract message metadata
  688.  tmp = to_string(.message) ?? ""
  689.  
  690.  ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  691.  if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  692.  if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  693.  if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  694.  if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  695.  if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  696.  
  697.  # we match only full IPv6 addresses
  698.  ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  699.  if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  700.  if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  701.  if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  702.  if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  703.  if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  704.  
  705.  numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  706.  if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  707.  if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  708.  if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  709.  if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  710.  if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  711.  if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  712.  if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  713.  if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  714.  if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  715.  if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  716.  
  717. } else {
  718.  del(.mysql)
  719. }
  720.  
  721. .mongodb = object!(parse_json(.message) ?? {})
  722.  
  723. if .mongodb != {} && exists(.mongodb.t."$$date") {
  724.  .platform = "MongoDB"
  725.  .dt = .mongodb.t."$$date"
  726.  .message = del(.mongodb.msg)
  727.  
  728.  if .mongodb.s == "I" {
  729.    .level = "info"
  730.  } else if .mongodb.s == "W" {
  731.    .level = "warning"
  732.  } else if .mongodb.s == "E" {
  733.    .level = "error"
  734.  } else if .mongodb.s == "F" {
  735.    .level = "fatal"
  736.  } else if .mongodb.s == "D" {
  737.    .level = "debug"
  738.  } else if .mongodb.s == "D1" {
  739.    .level = "debug1"
  740.  } else if .mongodb.s == "D2" {
  741.    .level = "debug2"
  742.  } else if .mongodb.s == "D3" {
  743.    .level = "debug3"
  744.  } else if .mongodb.s == "D4" {
  745.    .level = "debug4"
  746.  } else if .mongodb.s == "D5" {
  747.    .level = "debug5"
  748.  }
  749.  
  750.  # extract message metadata
  751.  tmp = to_string(.message) ?? ""
  752.  
  753.  ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  754.  if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  755.  if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  756.  if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  757.  if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  758.  if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  759.  
  760.  # we match only full IPv6 addresses
  761.  ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  762.  if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  763.  if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  764.  if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  765.  if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  766.  if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  767.  
  768.  numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  769.  if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  770.  if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  771.  if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  772.  if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  773.  if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  774.  if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  775.  if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  776.  if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  777.  if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  778.  if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  779.  
  780. } else {
  781.  del(.mongodb)
  782. }
  783.  
  784. .redis = parse_regex(.message, r'^(?P<pid>\d+):(?P<role_char>\w)\s+(?P<dt>\d+ \w+ \d+ \d{2}:\d{2}:\d{2}.\d+) (?P<level_symbol>.) (?P<message>.*)') ??
  785.    parse_regex(.message, r'^(?P<pid>\d+):(?P<role>\S+) \((?P<unix_timestamp>\d+)\) (?P<message>.*)') ??
  786.    {}
  787.  
  788. if .redis != {} {
  789.  .platform = "Redis"
  790.  if .redis.role_char == "M" {
  791.    .redis.role = "master"
  792.  } else if .redis.role_char == "S" {
  793.    .redis.role = "slave"
  794.  } else if .redis.role_char == "C" {
  795.    .redis.role = "RDB/AOF writing child"
  796.  } else if .redis.role_char == "X" {
  797.    .redis.role = "sentinel"
  798.  }
  799.  
  800.  if .redis.level_symbol == "." {
  801.    .level = "debug"
  802.  } else if .redis.level_symbol == "-" {
  803.    .level = "verbose"
  804.  } else if .redis.level_symbol == "*" {
  805.    .level = "notice"
  806.  } else if .redis.level_symbol == "#" {
  807.    .level = "warning"
  808.  }
  809.  
  810.  .redis.pid = to_int(.redis.pid) ?? null
  811.  
  812.  if exists(.redis.dt) {
  813.    .dt = format_timestamp!(parse_timestamp(del(.redis.dt), "%d %b %Y %T%.f") ?? now(), "%+")
  814.  } else if exists(.redis.unix_timestamp) {
  815.    .dt = format_timestamp!(to_timestamp(del(.redis.unix_timestamp)) ?? now(), "%+")
  816.  }
  817.  
  818.  .message = del(.redis.message)
  819.  
  820.  # extract message metadata
  821.  tmp = to_string(.message) ?? ""
  822.  
  823.  ips = parse_regex_all!(tmp, r'\b(?P<ip>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\b')
  824.  if exists(ips[0].ip) { .message_metadata.ipv4_1 = ips[0].ip; tmp = replace(tmp, string!(ips[0].ip), "") }
  825.  if exists(ips[1].ip) { .message_metadata.ipv4_2 = ips[1].ip; tmp = replace(tmp, string!(ips[1].ip), "") }
  826.  if exists(ips[2].ip) { .message_metadata.ipv4_3 = ips[2].ip; tmp = replace(tmp, string!(ips[2].ip), "") }
  827.  if exists(ips[3].ip) { .message_metadata.ipv4_4 = ips[3].ip; tmp = replace(tmp, string!(ips[3].ip), "") }
  828.  if exists(ips[4].ip) { .message_metadata.ipv4_5 = ips[4].ip; tmp = replace(tmp, string!(ips[4].ip), "") }
  829.  
  830.  # we match only full IPv6 addresses
  831.  ipv6s = parse_regex_all!(tmp, r'\b(?P<ip>(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4})\b')
  832.  if exists(ipv6s[0].ip) { .message_metadata.ipv6_1 = ipv6s[0].ip; tmp = replace(tmp, string!(ipv6s[0].ip), "") }
  833.  if exists(ipv6s[1].ip) { .message_metadata.ipv6_2 = ipv6s[1].ip; tmp = replace(tmp, string!(ipv6s[1].ip), "") }
  834.  if exists(ipv6s[2].ip) { .message_metadata.ipv6_3 = ipv6s[2].ip; tmp = replace(tmp, string!(ipv6s[2].ip), "") }
  835.  if exists(ipv6s[3].ip) { .message_metadata.ipv6_4 = ipv6s[3].ip; tmp = replace(tmp, string!(ipv6s[3].ip), "") }
  836.  if exists(ipv6s[4].ip) { .message_metadata.ipv6_5 = ipv6s[4].ip; tmp = replace(tmp, string!(ipv6s[4].ip), "") }
  837.  
  838.  numbers = parse_regex_all!(tmp, r'(?P<num>\b\d+(?:\.\d+)?\b)')
  839.  if exists(numbers[0].num) { .message_metadata.param1 = to_int(numbers[0].num) ?? to_float(numbers[0].num) ?? null }
  840.  if exists(numbers[1].num) { .message_metadata.param2 = to_int(numbers[1].num) ?? to_float(numbers[1].num) ?? null }
  841.  if exists(numbers[2].num) { .message_metadata.param3 = to_int(numbers[2].num) ?? to_float(numbers[2].num) ?? null }
  842.  if exists(numbers[3].num) { .message_metadata.param4 = to_int(numbers[3].num) ?? to_float(numbers[3].num) ?? null }
  843.  if exists(numbers[4].num) { .message_metadata.param5 = to_int(numbers[4].num) ?? to_float(numbers[4].num) ?? null }
  844.  if exists(numbers[5].num) { .message_metadata.param6 = to_int(numbers[5].num) ?? to_float(numbers[5].num) ?? null }
  845.  if exists(numbers[6].num) { .message_metadata.param7 = to_int(numbers[6].num) ?? to_float(numbers[6].num) ?? null }
  846.  if exists(numbers[7].num) { .message_metadata.param8 = to_int(numbers[7].num) ?? to_float(numbers[7].num) ?? null }
  847.  if exists(numbers[8].num) { .message_metadata.param9 = to_int(numbers[8].num) ?? to_float(numbers[8].num) ?? null }
  848.  if exists(numbers[9].num) { .message_metadata.param10 = to_int(numbers[9].num) ?? to_float(numbers[9].num) ?? null }
  849.  
  850. } else {
  851.  del(.redis)
  852. }
  853. '''
  854.  
  855. [sources.logtail_other_LmXXEqcihhTE5Kf67JPqA8Wk]
  856. type = "file"
  857. read_from = "beginning"
  858. ignore_older_secs = 600
  859. include = [
  860.   "/var/log/*.log",
  861.   "/var/log/**/*.log"
  862. ]
  863. exclude = [
  864.   "/var/log/apache2/*.log",
  865.   "/var/log/nginx/*.log",
  866.   "/var/log/postgresql/*.log",
  867.   "/var/log/mysql/*.log",
  868.   "/var/log/mongodb/*.log",
  869.   "/var/log/redis/*.log",
  870.   "/var/log/auth.log",
  871.   "/var/log/ufw.log",
  872.   "/var/log/docker/*.log"
  873. ]
  874.  
  875. [sinks.logtail_http_sink_LmXXEqcihhTE5Kf67JPqA8Wk]
  876. type = "http"
  877. uri = "https://in.logtail.com/"
  878. encoding.codec = "json"
  879. auth.strategy = "bearer"
  880. auth.token = "LmXXEqcihhTE5Kf67JPqA8Wk"
  881. inputs = [
  882.   "logtail_apache2_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  883.   "logtail_nginx_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  884.   "logtail_postgresql_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  885.   "logtail_mysql_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  886.   "logtail_mongodb_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  887.   "logtail_redis_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  888.   "logtail_auth_log_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  889.   "logtail_ufw_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  890.   "logtail_docker_parser_LmXXEqcihhTE5Kf67JPqA8Wk",
  891.   "logtail_other_LmXXEqcihhTE5Kf67JPqA8Wk"
  892. ]
  893.  
  894. # --- end of 2021-09-23: https://logtail.com/vector-toml/ubuntu/LmXXEqcihhTE5Kf67JPqA8Wk
  895.  
  896. root@techiebraj:~#
RAW Paste Data