API tools faq
paste
Advertisement
BrashEndeavours

Untitled

BrashEndeavours
Nov 24th, 2015
105
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 13.80 KB | None | 0 0
raw download clone embed print report
  1. cat /nsm/logstash/config/debug/testing.conf
  2. input {
  3. #stdin {
  4. # type => "BRO_connlog"
  5. #}
  6. #Production Logs#############################
  7. # file {
  8. # type => "BRO_httplog"
  9. # path => "/nsm/bro/logs/current/http.log"
  10. # start_position => "beginning"
  11. # sincedb_path => "/dev/null"
  12. # }
  13. # file {
  14. # type => "BRO_known_certslog"
  15. # path => "/nsm/bro/logs/current/known_certs.log"
  16. # start_position => "beginning"
  17. # sincedb_path => "/dev/null"
  18. # }
  19. # file {
  20. # type => "BRO_noticelog"
  21. # path => "/nsm/bro/logs/current/notice.log"
  22. # start_position => "beginning"
  23. # sincedb_path => "/dev/null"
  24. # }
  25. # file {
  26. # type => "BRO_known_hostslog"
  27. # path => "/nsm/bro/logs/current/known_hosts.log"
  28. # start_position => "beginning"
  29. # sincedb_path => "/dev/null"
  30. # }
  31. # file {
  32. # type => "BRO_known_serviceslog"
  33. # path => "/nsm/bro/logs/current/known_services.log"
  34. # start_position => "beginning"
  35. # sincedb_path => "/dev/null"
  36. # }
  37. # file {
  38. # type => "BRO_sshlog"
  39. # path => "/nsm/bro/logs/current/ssh.log"
  40. # start_position => "beginning"
  41. # sincedb_path => "/dev/null"
  42. # }
  43. # file {
  44. # type => "BRO_dpdlog"
  45. # path => "/nsm/bro/logs/current/dpd.log"
  46. # start_position => "beginning"
  47. # sincedb_path => "/dev/null"
  48. # }
  49. file {
  50. type => "BRO_connlog"
  51. path => "/nsm/bro/logs/current/conn.log"
  52. start_position => "beginning"
  53. sincedb_path => "/dev/null"
  54. }
  55. # file {
  56. # type => "BRO_weirdlog"
  57. # path => "/nsm/bro/logs/current/weird.log"
  58. # start_position => "beginning"
  59. # sincedb_path => "/dev/null"
  60. # }
  61. # file {
  62. # type => "BRO_app_statslog"
  63. # path => "/nsm/bro/logs/current/appstats.log"
  64. # start_position => "beginning"
  65. # sincedb_path => "/dev/null"
  66. # }
  67. # file {
  68. # type => "BRO_dhcplog"
  69. # path => "/nsm/bro/logs/current/dhcp.log"
  70. # start_position => "beginning"
  71. # sincedb_path => "/dev/null"
  72. # }
  73. # file {
  74. # type => "BRO_fileslog"
  75. # path => "/nsm/bro/logs/current/files.log"
  76. # start_position => "beginning"
  77. # sincedb_path => "/dev/null"
  78. # }
  79. # file {
  80. # type => "BRO_ssllog"
  81. # path => "/nsm/bro/logs/current/ssl.log"
  82. # start_position => "beginning"
  83. # sincedb_path => "/dev/null"
  84. # }
  85. # file {
  86. # type => "BRO_noticelog"
  87. # path => "/nsm/bro/logs/current/notice.log"
  88. # start_position => "beginning"
  89. # sincedb_path => "/dev/null"
  90. # }
  91. # file {
  92. # type => "BRO_softwarelog"
  93. # path => "/nsm/bro/logs/current/software.log"
  94. # start_position => "beginning"
  95. # sincedb_path => "/dev/null"
  96. # }
  97. # file {
  98. # type => "BRO_dnslog"
  99. # path => "/nsm/bro/logs/current/dns.log"
  100. # start_position => "beginning"
  101. # sincedb_path => "/dev/null"
  102. # }
  103. # file {
  104. # type => "BRO_intellog"
  105. # path => "/nsm/bro/logs/current/intel.log"
  106. # start_position => "beginning"
  107. # sincedb_path => "/dev/null"
  108. # }
  109. }
  110.  
  111. filter {
  112. if [message] =~ /^#/ {
  113. drop { }
  114. }
  115. else {
  116.  
  117. # BRO_app_statslog ######################
  118. # if [type] == "BRO_app_statslog" {
  119. # grok {
  120. # match => [ "message", "(?<ts>(.*?))\t(?<ts_delta>(.*?))\t(?<app>(.*?))\t(?<uniq_hosts>(.*?))\t(?<hits>(.*?))\t(?<bytes>(.*))" ]
  121. # }
  122. # }
  123.  
  124. # BRO_connlog ######################
  125. if [type] == "BRO_connlog" {
  126. grok {
  127. match => [
  128. "message", "%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents}"
  129. #"message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(?<tunnel_parents>(.*?))\t(?<orig_cc>(.*?))\t(?<resp_cc>(.*?))\t(?<sensorname>(.*))",
  130. #"message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<service>(.*?))\t(?<duration>(.*?))\t(?<orig_bytes>(.*?))\t(?<resp_bytes>(.*?))\t(?<conn_state>(.*?))\t(?<local_orig>(.*?))\t(?<missed_bytes>(.*?))\t(?<history>(.*?))\t(?<orig_pkts>(.*?))\t(?<orig_ip_bytes>(.*?))\t(?<resp_pkts>(.*?))\t(?<resp_ip_bytes>(.*?))\t(%{NOTSPACE:tunnel_parents})"
  131. ]
  132. }
  133. }
  134.  
  135. # BRO_noticelog ######################
  136. # if [type] == "BRO_noticelog" {
  137. # grok {
  138. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<fuid>(.*?))\t(?<file_mime_type>(.*?))\t(?<file_desc>(.*?))\t(?<proto>(.*?))\t(?<note>(.*?))\t(?<msg>(.*?))\t(?<sub>(.*?))\t(?<src>(.*?))\t(?<dst>(.*?))\t(?<p>(.*?))\t(?<n>(.*?))\t(?<peer_descr>(.*?))\t(?<actions>(.*?))\t(?<suppress_for>(.*?))\t(?<dropped>(.*?))\t(?<remote_location_country_code>(.*?))\t(?<remote_location_region>(.*?))\t(?<remote_location_city>(.*?))\t(?<remote_location_latitude>(.*?))\t(?<remote_location_longitude>(.*))" ]
  139. # }
  140. # }
  141.  
  142.  
  143. # BRO_dhcplog ######################
  144. # if [type] == "BRO_dhcplog" {
  145. # grok {
  146. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<mac>(.*?))\t(?<assigned_ip>(.*?))\t(?<lease_time>(.*?))\t(?<trans_id>(.*))" ]
  147. # }
  148. # }
  149.  
  150. # BRO_dnslog ######################
  151. # if [type] == "BRO_dnslog" {
  152. # grok {
  153. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<trans_id>(.*?))\t(?<query>(.*?))\t(?<qclass>(.*?))\t(?<qclass_name>(.*?))\t(?<qtype>(.*?))\t(?<qtype_name>(.*?))\t(?<rcode>(.*?))\t(?<rcode_name>(.*?))\t(?<AA>(.*?))\t(?<TC>(.*?))\t(?<RD>(.*?))\t(?<RA>(.*?))\t(?<Z>(.*?))\t(?<answers>(.*?))\t(?<TTLs>(.*?))\t(?<rejected>(.*))" ]
  154. # }
  155. # }
  156.  
  157. # BRO_softwarelog ######################
  158. # if [type] == "BRO_softwarelog" {
  159. # grok {
  160. # match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<host_p>(.*?))\t(?<software_type>(.*?))\t(?<name>(.*?))\t(?<version_major>(.*?))\t(?<version_minor>(.*?))\t(?<version_minor2>(.*?))\t(?<version_minor3>(.*?))\t(?<version_addl>(.*?))\t(?<unparsed_version>(.*))" ]
  161. # }
  162. # }
  163.  
  164. # BRO_dpdlog ######################
  165. # if [type] == "BRO_dpdlog" {
  166. # grok {
  167. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<proto>(.*?))\t(?<analyzer>(.*?))\t(?<failure_reason>(.*))" ]
  168. # }
  169. # }
  170.  
  171. # BRO_fileslog ######################
  172. # if [type] == "BRO_fileslog" {
  173. # grok {
  174. # match => [ "message", "(?<ts>(.*?))\t(?<fuid>(.*?))\t(?<tx_hosts>(.*?))\t(?<rx_hosts>(.*?))\t(?<conn_uids>(.*?))\t(?<source>(.*?))\t(?<depth>(.*?))\t(?<analyzers>(.*?))\t(?<mime_type>(.*?))\t(?<filename>(.*?))\t(?<duration>(.*?))\t(?<local_orig>(.*?))\t(?<is_orig>(.*?))\t(?<seen_bytes>(.*?))\t(?<total_bytes>(.*?))\t(?<missing_bytes>(.*?))\t(?<overflow_bytes>(.*?))\t(?<timedout>(.*?))\t(?<parent_fuid>(.*?))\t(?<md5>(.*?))\t(?<sha1>(.*?))\t(?<sha256>(.*?))\t(?<extracted>(.*))" ]
  175. # }
  176. # }
  177.  
  178. # BRO_httplog ######################
  179. # if [type] == "BRO_httplog" {
  180. # grok {
  181. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<trans_depth>(.*?))\t(?<method>(.*?))\t(?<bro_host>(.*?))\t(?<uri>(.*?))\t(?<referrer>(.*?))\t(?<user_agent>(.*?))\t(?<request_body_len>(.*?))\t(?<response_body_len>(.*?))\t(?<status_code>(.*?))\t(?<status_msg>(.*?))\t(?<info_code>(.*?))\t(?<info_msg>(.*?))\t(?<filename>(.*?))\t(?<http_tags>(.*?))\t(?<username>(.*?))\t(?<password>(.*?))\t(?<proxied>(.*?))\t(?<orig_fuids>(.*?))\t(?<orig_mime_types>(.*?))\t(?<resp_fuids>(.*?))\t(?<resp_mime_types>(.*))" ]
  182. # }
  183. # }
  184.  
  185. # BRO_known_certslog ######################
  186. # if [type] == "BRO_known_certslog" {
  187. # grok {
  188. # match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<port_num>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<serial>(.*))" ]
  189. # }
  190. # }
  191.  
  192. # BRO_known_hostslog ######################
  193. # if [type] == "BRO_known_hostslog" {
  194. # grok {
  195. # match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*))" ]
  196. # }
  197. # }
  198.  
  199. # BRO_known_serviceslog ######################
  200. # if [type] == "BRO_known_serviceslog" {
  201. # grok {
  202. # match => [ "message", "(?<ts>(.*?))\t(?<bro_host>(.*?))\t(?<port_num>(.*?))\t(?<port_proto>(.*?))\t(?<service>(.*))" ]
  203. # }
  204. # }
  205.  
  206. # BRO_sshlog ######################
  207. # if [type] == "BRO_sshlog" {
  208. # grok {
  209. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<status>(.*?))\t(?<direction>(.*?))\t(?<client>(.*?))\t(?<server>(.*?))\t(?<remote_location_country_code>(.*?))\t(?<remote_location_region>(.*?))\t(?<remote_location_city>(.*?))\t(?<remote_location_latitude>(.*?))\t(?<remote_location_longitude>(.*))" ]
  210. # }
  211. # }
  212.  
  213. # BRO_ssllog ######################
  214. # if [type] == "BRO_ssllog" {
  215. # grok {
  216. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<version>(.*?))\t(?<cipher>(.*?))\t(?<server_name>(.*?))\t(?<session_id>(.*?))\t(?<subject>(.*?))\t(?<issuer_subject>(.*?))\t(?<not_valid_before>(.*?))\t(?<not_valid_after>(.*?))\t(?<last_alert>(.*?))\t(?<client_subject>(.*?))\t(?<client_issuer_subject>(.*?))\t(?<cert_hash>(.*?))\t(?<validation_status>(.*))" ]
  217. # }
  218. # }
  219.  
  220. # BRO_weirdlog ######################
  221. # if [type] == "BRO_weirdlog" {
  222. # grok {
  223. # match => [ "message", "(?<ts>(.*?))\t(?<uid>(.*?))\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t(?<name>(.*?))\t(?<addl>(.*?))\t(?<notice>(.*?))\t(?<peer>(.*))" ]
  224. # }
  225. # }
  226. # if [type]== "BRO_intellog" {
  227. # grok {
  228. # match => [ "message", "(?<ts>(.*?))\t%{DATA:uid}\t(?<id_orig_h>(.*?))\t(?<id_orig_p>(.*?))\t(?<id_resp_h>(.*?))\t(?<id_resp_p>(.*?))\t%{DATA:fuid}\t%{DATA:file_mime_type}\t%{DATA:file_desc}\t(?<seen_indicator>(.*?))\t(?<seen_indicator_type>(.*?))\t(?<seen.where>(.*?))\t%{NOTSPACE:sources}" ]
  229. # }
  230. # }
  231. }
  232. date {
  233. match => [ "ts", "UNIX" ]
  234. }
  235. }
  236. #filter {
  237. # if [bro_host] {
  238. # mutate {
  239. # replace => [ "host", "%{bro_host}" ]
  240. # }
  241. # }
  242. #}
  243. #filter {
  244. # if "BRO" in [type] {
  245. # if [id_orig_h] {
  246. # mutate {
  247. # add_field => [ "senderbase_lookup", "http://www.senderbase.org/lookup/?search_string=%{id_orig_h}" ]
  248. # add_field => [ "CBL_lookup", "http://cbl.abuseat.org/lookup.cgi?ip=%{id_orig_h}" ]
  249. # add_field => [ "Spamhaus_lookup", "http://www.spamhaus.org/query/bl?ip=%{id_orig_h}" ]
  250. # }
  251. # }
  252. # mutate {
  253. # add_tag => [ "BRO" ]
  254. # }
  255. # mutate {
  256. # convert => [ "id_orig_p", "integer" ]
  257. # convert => [ "id_resp_p", "integer" ]
  258. # convert => [ "orig_bytes", "integer" ]
  259. # convert => [ "resp_bytes", "integer" ]
  260. # convert => [ "missed_bytes", "integer" ]
  261. # convert => [ "orig_pkts", "integer" ]
  262. # convert => [ "orig_ip_bytes", "integer" ]
  263. # convert => [ "resp_pkts", "integer" ]
  264. # convert => [ "resp_ip_bytes", "integer" ]
  265. # }
  266. # }
  267. #}
  268. filter {
  269. if [type] == "BRO_connlog" {
  270. #The following makes use of the translate filter (logstash contrib) to convert conn_state into human text. Saves having to look up values for packet introspection
  271. translate {
  272. field => "conn_state"
  273. destination => "conn_state_full"
  274. dictionary => [
  275. "S0", "Connection attempt seen, no reply",
  276. "S1", "Connection established, not terminated",
  277. "S2", "Connection established and close attempt by originator seen (but no reply from responder)",
  278. "S3", "Connection established and close attempt by responder seen (but no reply from originator)",
  279. "SF", "Normal SYN/FIN completion",
  280. "REJ", "Connection attempt rejected",
  281. "RSTO", "Connection established, originator aborted (sent a RST)",
  282. "RSTR", "Established, responder aborted",
  283. "RSTOS0", "Originator sent a SYN followed by a RST, we never saw a SYN-ACK from the responder",
  284. "RSTRH", "Responder sent a SYN ACK followed by a RST, we never saw a SYN from the (purported) originator",
  285. "SH", "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was half open)",
  286. "SHR", "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator",
  287. "OTH", "No SYN seen, just midstream traffic (a partial'partial connection' that was not later closed)"
  288. ]
  289. }
  290. }
  291. }
  292.  
  293. # Benchmark!
  294. filter {
  295. metrics {
  296. meter => "benchmark"
  297. add_tag => "metric"
  298. }
  299. }
  300.  
  301. output {
  302. if "metric" in [tags] {
  303. stdout {
  304. codec => line {
  305. format => "1m rate: %{[benchmark][rate_1m]} ( %{[benchmark][count]} )"
  306. }
  307. }
  308. }
  309. stdout { codec => rubydebug }
  310. # elasticsearch {
  311. # hosts => ["127.0.0.1:9200"]
  312. # }
  313. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!