API tools faq
paste
Advertisement
BrashEndeavours

logstash_bummer

BrashEndeavours
Nov 24th, 2015
130
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 13.03 KB | None | 0 0
raw download clone embed print report
  1. #Production Logs#############################
  2. file {
  3. type => "BRO_httplog"
  4. path => "/nsm/bro/logs/current/http.log"
  5. start_position => "beginning"
  6. sincedb_path => "/dev/null"
  7. }
  8. file {
  9. type => "BRO_known_certslog"
  10. path => "/nsm/bro/logs/current/known_certs.log"
  11. start_position => "beginning"
  12. sincedb_path => "/dev/null"
  13. }
  14. file {
  15. type => "BRO_noticelog"
  16. path => "/nsm/bro/logs/current/notice.log"
  17. start_position => "beginning"
  18. sincedb_path => "/dev/null"
  19. }
  20. file {
  21. type => "BRO_known_hostslog"
  22. path => "/nsm/bro/logs/current/known_hosts.log"
  23. start_position => "beginning"
  24. sincedb_path => "/dev/null"
  25. }
  26. file {
  27. type => "BRO_known_serviceslog"
  28. path => "/nsm/bro/logs/current/known_services.log"
  29. start_position => "beginning"
  30. sincedb_path => "/dev/null"
  31. }
  32. file {
  33. type => "BRO_sshlog"
  34. path => "/nsm/bro/logs/current/ssh.log"
  35. start_position => "beginning"
  36. sincedb_path => "/dev/null"
  37. }
  38. file {
  39. type => "BRO_dpdlog"
  40. path => "/nsm/bro/logs/current/dpd.log"
  41. start_position => "beginning"
  42. sincedb_path => "/dev/null"
  43. }
  44. file {
  45. type => "BRO_connlog"
  46. path => "/nsm/bro/logs/current/conn.log"
  47. start_position => "beginning"
  48. sincedb_path => "/dev/null"
  49. }
  50. file {
  51. type => "BRO_weirdlog"
  52. path => "/nsm/bro/logs/current/weird.log"
  53. start_position => "beginning"
  54. sincedb_path => "/dev/null"
  55. }
  56. file {
  57. type => "BRO_app_statslog"
  58. path => "/nsm/bro/logs/current/appstats.log"
  59. start_position => "beginning"
  60. sincedb_path => "/dev/null"
  61. }
  62. file {
  63. type => "BRO_dhcplog"
  64. path => "/nsm/bro/logs/current/dhcp.log"
  65. start_position => "beginning"
  66. sincedb_path => "/dev/null"
  67. }
  68. file {
  69. type => "BRO_fileslog"
  70. path => "/nsm/bro/logs/current/files.log"
  71. start_position => "beginning"
  72. sincedb_path => "/dev/null"
  73. }
  74. file {
  75. type => "BRO_ssllog"
  76. path => "/nsm/bro/logs/current/ssl.log"
  77. start_position => "beginning"
  78. sincedb_path => "/dev/null"
  79. }
  80. file {
  81. type => "BRO_noticelog"
  82. path => "/nsm/bro/logs/current/notice.log"
  83. start_position => "beginning"
  84. sincedb_path => "/dev/null"
  85. }
  86. file {
  87. type => "BRO_softwarelog"
  88. path => "/nsm/bro/logs/current/software.log"
  89. start_position => "beginning"
  90. sincedb_path => "/dev/null"
  91. }
  92. file {
  93. type => "BRO_dnslog"
  94. path => "/nsm/bro/logs/current/dns.log"
  95. start_position => "beginning"
  96. sincedb_path => "/dev/null"
  97. }
  98. file {
  99. type => "BRO_intellog"
  100. path => "/nsm/bro/logs/current/intel.log"
  101. start_position => "beginning"
  102. sincedb_path => "/dev/null"
  103. }
  104. }
  105.  
  106. filter {
  107. if [message] =~ /^#/ {
  108. drop { }
  109. }
  110. else {
  111.  
  112. # BRO_app_statslog ######################
  113. if [type] == "BRO_app_statslog" {
  114. grok {
  115. match => [ "message", "(?<ts>(.*?))\t(?<ts_delta>(.*?))\t(?<app>(.*?))\t(?<uniq_hosts>(.*?))\t(?<hits>(.*?))\t(?<bytes>(.*))" ]
  116. }
  117. }
  118.  
  119. # BRO_connlog ######################
  120. if [type] == "BRO_connlog" {
  121. grok {
  122. match => [
  123. "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(?<tunnel_parents>(.*?))\t(?<orig_cc>(.*?))\t(?<resp_cc>(.*?))\t(?<sensorname>(.*))",
  124. "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(%{NOTSPACE:tunnel_parents})"
  125. ]
  126. }
  127. }
  128.  
  129. # BRO_noticelog ######################
  130. if [type] == "BRO_noticelog" {
  131. grok {
  132. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<fuid>(.*?))\t(?<file_mime_type>(.*?))\t(?<file_desc>(.*?))\t(?<proto>(.*?))\t(?<note>(.*?))\t(?<msg>(.*?))\t(?<sub>(.*?))\t(?<src>(.*?))\t(?<dst>(.*?))\t(?<p>(.*?))\t(?<n>(.*?))\t(?<peer_descr>(.*?))\t(?<actions>(.*?))\t(?<suppress_for>(.*?))\t(?<dropped>(.*?))\t(?<remote_location_country_code>(.*?))\t(?<remote_location_region>(.*?))\t(?<remote_location_city>(.*?))\t(?<remote_location_latitude>(.*?))\t(?<remote_location_longitude>(.*))" ]
  133. }
  134. }
  135.  
  136.  
  137. # BRO_dhcplog ######################
  138. if [type] == "BRO_dhcplog" {
  139. grok {
  140. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<mac>(.*?))\t(?<assigned_ip>(.*?))\t(?<lease_time>(.*?))\t(?<trans_id>(.*))" ]
  141. }
  142. }
  143.  
  144. # BRO_dnslog ######################
  145. if [type] == "BRO_dnslog" {
  146. grok {
  147. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<trans_id>(.*?))\t(?<query>(.*?))\t(?<qclass>(.*?))\t(?<qclass_name>(.*?))\t(?<qtype>(.*?))\t(?<qtype_name>(.*?))\t(?<rcode>(.*?))\t(?<rcode_name>(.*?))\t(?<AA>(.*?))\t(?<TC>(.*?))\t(?<RD>(.*?))\t(?<RA>(.*?))\t(?<Z>(.*?))\t(?<answers>(.*?))\t(?<TTLs>(.*?))\t(?<rejected>(.*))" ]
  148. }
  149. }
  150.  
  151. # BRO_softwarelog ######################
  152. if [type] == "BRO_softwarelog" {
  153. grok {
  154. match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<host_p>(.*?))\t(?<software_type>(.*?))\t(?<name>(.*?))\t(?<version_major>(.*?))\t(?<version_minor>(.*?))\t(?<version_minor2>(.*?))\t(?<version_minor3>(.*?))\t(?<version_addl>(.*?))\t(?<unparsed_version>(.*))" ]
  155. }
  156. }
  157.  
  158. # BRO_dpdlog ######################
  159. if [type] == "BRO_dpdlog" {
  160. grok {
  161. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<analyzer>(.*?))\t(?<failure_reason>(.*))" ]
  162. }
  163. }
  164.  
  165. # BRO_fileslog ######################
  166. if [type] == "BRO_fileslog" {
  167. grok {
  168. match => [ "message", "(?<ts>(.*?))\t(?<fuid>(.*?))\t(?<tx_hosts>(.*?))\t(?<rx_hosts>(.*?))\t(?<conn_uids>(.*?))\t(?<source>(.*?))\t(?<depth>(.*?))\t(?<analyzers>(.*?))\t(?<mime_type>(.*?))\t(?<filename>(.*?))\t(?<duration>(.*?))\t(?<local_orig>(.*?))\t(?<is_orig>(.*?))\t(?<seen_bytes>(.*?))\t(?<total_bytes>(.*?))\t(?<missing_bytes>(.*?))\t(?<overflow_bytes>(.*?))\t(?<timedout>(.*?))\t(?<parent_fuid>(.*?))\t(?<md5>(.*?))\t(?<sha1>(.*?))\t(?<sha256>(.*?))\t(?<extracted>(.*))" ]
  169. }
  170. }
  171.  
  172. # BRO_httplog ######################
  173. if [type] == "BRO_httplog" {
  174. grok {
  175. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<bro_host>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<user_agent>(.*?))\t(?<request_body_len>(.*?))\t(?<response_body_len>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<info_code>(.*?))\t(?<info_msg>(.*?))\t(?<filename>(.*?))\t(?<http_tags>(.*?))\t(?<username>(.*?))\t(?<password>(.*?))\t(?<proxied>(.*?))\t(?<orig_fuids>(.*?))\t(?<orig_mime_types>(.*?))\t(?<resp_fuids>(.*?))\t(?<resp_mime_types>(.*))" ]
  176. }
  177. }
  178.  
  179. # BRO_known_certslog ######################
  180. if [type] == "BRO_known_certslog" {
  181. grok {
  182. match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<port_num>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<serial>(.*))" ]
  183. }
  184. }
  185.  
  186. # BRO_known_hostslog ######################
  187. if [type] == "BRO_known_hostslog" {
  188. grok {
  189. match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*))" ]
  190. }
  191. }
  192.  
  193. # BRO_known_serviceslog ######################
  194. if [type] == "BRO_known_serviceslog" {
  195. grok {
  196. match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<port_num>(.*?))\t(?<port_proto>(.*?))\t(?<service>(.*))" ]
  197. }
  198. }
  199.  
  200. # BRO_sshlog ######################
  201. if [type] == "BRO_sshlog" {
  202. grok {
  203. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<status>(.*?))\t(?<direction>(.*?))\t(?<client>(.*?))\t(?<server>(.*?))\t(?<remote_location_country_code>(.*?))\t(?<remote_location_region>(.*?))\t(?<remote_location_city>(.*?))\t(?<remote_location_latitude>(.*?))\t(?<remote_location_longitude>(.*))" ]
  204. }
  205. }
  206.  
  207. # BRO_ssllog ######################
  208. if [type] == "BRO_ssllog" {
  209. grok {
  210. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<version>(.*?))\t(?<cipher>(.*?))\t(?<server_name>(.*?))\t(?<session_id>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<not_valid_before>(.*?))\t(?<not_valid_after>(.*?))\t(?<last_alert>(.*?))\t(?<client_subject>(.*?))\t(?<client_issuer_subject>(.*?))\t(?<cert_hash>(.*?))\t(?<validation_status>(.*))" ]
  211. }
  212. }
  213.  
  214. # BRO_weirdlog ######################
  215. if [type] == "BRO_weirdlog" {
  216. grok {
  217. match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<name>(.*?))\t(?<addl>(.*?))\t(?<notice>(.*?))\t(?<peer>(.*))" ]
  218. }
  219. }
  220. if [type]== "BRO_intellog" {
  221. grok {
  222. match => [ "message", "(?<ts>(.*?))\t%{DATA:uid}\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t%{DATA:fuid}\t%{DATA:file_mime_type}\t%{DATA:file_desc}\t(?<seen_indicator>(.*?))\t(?<seen_indicator_type>(.*?))\t(?<seen.where>(.*?))\t%{NOTSPACE:sources}" ]
  223. }
  224. }
  225. }
  226. date {
  227. match => [ "ts", "UNIX" ]
  228. }
  229. }
  230. filter {
  231. if [bro_host] {
  232. mutate {
  233. replace => [ "host", "%{bro_host}" ]
  234. }
  235. }
  236. }
  237. filter {
  238. if "BRO" in [type] {
  239. if [id_orig_h] {
  240. mutate {
  241. add_field => [ "senderbase_lookup", "http://www.senderbase.org/lookup/?search_string=%{id_orig_h}" ]
  242. add_field => [ "CBL_lookup", "http://cbl.abuseat.org/lookup.cgi?ip=%{id_orig_h}" ]
  243. add_field => [ "Spamhaus_lookup", "http://www.spamhaus.org/query/bl?ip=%{id_orig_h}" ]
  244. }
  245. }
  246. mutate {
  247. add_tag => [ "BRO" ]
  248. }
  249. mutate {
  250. convert => [ "id_orig_p", "integer" ]
  251. convert => [ "id_resp_p", "integer" ]
  252. convert => [ "orig_bytes", "integer" ]
  253. convert => [ "resp_bytes", "integer" ]
  254. convert => [ "missed_bytes", "integer" ]
  255. convert => [ "orig_pkts", "integer" ]
  256. convert => [ "orig_ip_bytes", "integer" ]
  257. convert => [ "resp_pkts", "integer" ]
  258. convert => [ "resp_ip_bytes", "integer" ]
  259. }
  260. }
  261. }
  262. filter {
  263. if [type] == "BRO_connlog" {
  264. #The following makes use of the translate filter (logstash contrib) to convert conn_state into human text. Saves having to look up values for packet introspection
  265. translate {
  266. field => "conn_state"
  267. destination => "conn_state_full"
  268. dictionary => [
  269. "S0", "Connection attempt seen, no reply",
  270. "S1", "Connection established, not terminated",
  271. "S2", "Connection established and close attempt by originator seen (but no reply from responder)",
  272. "S3", "Connection established and close attempt by responder seen (but no reply from originator)",
  273. "SF", "Normal SYN/FIN completion",
  274. "REJ", "Connection attempt rejected",
  275. "RSTO", "Connection established, originator aborted (sent a RST)",
  276. "RSTR", "Established, responder aborted",
  277. "RSTOS0", "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder",
  278. "RSTRH", "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator",
  279. "SH", "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was half open)",
  280. "SHR", "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator",
  281. "OTH", "No SYN seen, just midstream traffic (a partial'partial connection' that was not later closed)"
  282. ]
  283. }
  284. }
  285. }
  286.  
  287. # Benchmark
  288. filter {
  289. metrics {
  290. meter => "benchmark"
  291. add_tag => "metric"
  292. }
  293. }
  294.  
  295. output {
  296. if "metric" in [tags] {
  297. stdout {
  298. codec => line {
  299. format => "1m rate: %{[benchmark][rate_1m]} ( %{[benchmark][count]} )"
  300. }
  301. }
  302. }
  303. }
  304. output {
  305. elasticsearch {
  306. hosts => ["127.0.0.1:9200"]
  307. }
  308. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!