API tools faq
paste
Advertisement
pup_seba

zimbra logstash filter

pup_seba
Sep 15th, 2022
281
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 11.00 KB | None | 0 0
raw download clone embed print report
  1. filter {
  2. ### general
  3. mutate { remove_field => ["host"] }
  4.  
  5. if "beats_input_codec_plain_applied" in [tags] {
  6. mutate { remove_tag => ["beats_input_codec_plain_applied"] }
  7. }
  8.  
  9. ### mailbox.log
  10. if [log][file][path] == "/opt/zimbra/log/mailbox.log" {
  11. grok {
  12. match => { 'message' => "^%{TIMESTAMP_ISO8601:logtime} %{LOGLEVEL:loglevel}[[:space:]]*%{GREEDYDATA:message}$" }
  13. overwrite => ["message"]
  14. }
  15.  
  16. date {
  17. match => [ "logtime", "yyyy-MM-dd HH:mm:ss,SSS" ]
  18. target => "@timestamp"
  19. locale => "en_US"
  20. tag_on_failure => ["date_filter_failed"]
  21. }
  22.  
  23. mutate { add_tag => [ "zimbra" ] }
  24.  
  25. ### message count
  26. if "smtp - Sending message to" in [message] {
  27. grok {
  28. patterns_dir => ["/etc/logstash/patterns"]
  29. #mailbox.log_1
  30. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};mid=[\d]*;oip=%{IP:oip};(port=[\d]*;)*ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*).*Message-ID=<(?<messageid>[\w$\.\@\-]*)>,.*sender=%{ZIMBRA_USER:sender}, nrcpts=(?<nrcpts>[\d]*),.*" }
  31. #mailbox.log_2
  32. match => { 'message' => "^\[LmtpServer-[\d]*\] \[name=%{ZIMBRA_USER};mid=[\d]*;ip=%{IP};\] smtp - Sending message to MTA at %{IPORHOST}.*Message-ID=<(?<messageid>[\w$\.\@\-]*)>,.*sender=%{ZIMBRA_USER:sender}, nrcpts=(?<nrcpts>[\d]*),.*" }
  33. #mailbox.log_18
  34. match => { 'message' => "^\[ScheduledTask-[\d]*\] \[\] smtp - Sending message to MTA at %{IPORHOST}.*Message-ID=<(?<messageid>[\w$\.\@\-]*)>,.*sender=%{ZIMBRA_USER:sender}, nrcpts=(?<nrcpts>[\d]*),.*$" }
  35. #mailbox.log_19
  36. match => { 'message' => "^\[ScheduledTask-[\d]*\] \[name=%{ZIMBRA_USER};mid=[\d]*;ds=[\w\-]*;\] smtp - Sending message to MTA at %{IPORHOST}.*Message-ID=<(?<messageid>[\w$\.\@\-]*)>, origMsgId=[\w\-:]*, replyType=[\w], .*sender=%{ZIMBRA_USER:sender}, nrcpts=(?<nrcpts>[\d]*),.*$" }
  37. remove_tag => [ "_grokparsefailure"]
  38. }
  39. mutate {
  40. add_tag => [ "message_sent" ]
  41. convert => { "nrcpts" => "integer" }
  42. }
  43. } ### message count
  44.  
  45. ### authentication failed
  46. if "authentication failed" in [message] {
  47. mutate { add_tag => [ "authentication_failed" ] }
  48.  
  49. if "invalid password" in [message] {
  50. mutate { add_tag => [ "invalid_password" ] }
  51. grok {
  52. patterns_dir => ["/etc/logstash/patterns"]
  53. #mailbox.log_3
  54. match => { 'message' => "^\[[\w\d\-\]]* \[ip=%{IP};(oip|ip)=%{IP:oip};via=.*;ua=%{ZIMBRA_CLIENT:client};cid=[\d]*;\] imap - authentication failed for \[%{ZIMBRA_USER:account}] \(invalid password\)$" }
  55. #mailbox.log_4
  56. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};oport=[\d]*;oproto=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*).*%{ZIMBRA_SOAP_HANDLER} \[%{ZIMBRA_USER}\], invalid password$" }
  57. #mailbox.log_5
  58. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);\].*%{ZIMBRA_SOAP_HANDLER} \[" }
  59. #mailbox.log_6
  60. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};port=[\d]*;ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);\].*%{ZIMBRA_SOAP_HANDLER} \[%{ZIMBRA_USER}\], invalid password$" }
  61. remove_tag => [ "_grokparsefailure"]
  62. }
  63. }
  64.  
  65. if "status is locked" in [message] {
  66. mutate {
  67. add_tag => [ "locked_attempts" ]
  68. }
  69. grok {
  70. patterns_dir => ["/etc/logstash/patterns"]
  71. #mailbox.log_7
  72. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip}.*oproto=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*).*%{ZIMBRA_SOAP_HANDLER} \[.*status is locked$" }
  73. remove_tag => ["_grokparsefailure"]
  74. }
  75. }
  76.  
  77. if "account not found" in [message] {
  78. mutate { add_tag => [ "account_not_found" ] }
  79. grok {
  80. patterns_dir => ["/etc/logstash/patterns"]
  81. #mailbox.log.8
  82. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[(oip|ip)=%{IP:oip}.*oproto=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);] SoapEngine - handler exception: authentication failed for \[%{ZIMBRA_USER}\], account not found$" }
  83. #mailbox.log_9
  84. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[(oip|ip)=%{IP:oip};ua=%{ZIMBRA_CLIENT:client};soapId(?<soapid>[^;]*);] %{ZIMBRA_SOAP_HANDLER} \[%{ZIMBRA_USER:account}], account not found$" }
  85. remove_tag => [ "_grokparsefailure"]
  86. }
  87. }
  88.  
  89. if "account lockout" in [message] {
  90. mutate { add_tag => [ "locked_account" ] }
  91. grok {
  92. patterns_dir => ["/etc/logstash/patterns"]
  93. #mailbox.log_10
  94. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=(?<account>[^;]+);(oip|ip)=%{IP:oip};ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);].*?((?<reason>(?=account lockout).*).*)?$" }
  95. #mailbox.log_11
  96. match => { 'message' => "^\[name=(?<validated_account>[^;]*);oip=%{IP:oip}.*oproto=(?<protocol>[^;]*).*soapId=(?<soapid>[^;]*).*\[(?<account>[^\]]*).*account lockout" }
  97. match => { 'message' => "^\[ImapSSLServer-[\d]*\] \[ip=%{IP:ip};oip=%{IP:oip}.*ua=[^ ]* (imap) - authentication failed for \[%{ZIMBRA_USER:account}\].* \((?<reason>account lockout)\)?$" }
  98. remove_tag => [ "_grokparsefailure"]
  99. }
  100. }
  101.  
  102. if "status is closed" in [message] {
  103. mutate { add_tag => [ "account_closed" ] }
  104. grok {
  105. patterns_dir => ["/etc/logstash/patterns"]
  106. #mailbox.log_12
  107. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};oport=[\d]*;oproto=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);\].*%{ZIMBRA_SOAP_HANDLER} \[(%{ZIMBRA_USER}|[\w]*)\], account\(or domain\) status is closed$" }
  108. remove_tag => [ "_grokparsefailure"]
  109. }
  110. }
  111. } ### authentication failed
  112.  
  113. ### dos filter triggered
  114. if "suspended, for repeated failed login" in [message] {
  115. mutate { add_tag => ["suspended_ip"] }
  116. grok {
  117. patterns_dir => ["/etc/logstash/patterns"]
  118. #mailbox.log_13
  119. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP}[\s\[\]\w\-]*misc - Access from IP *%{IP:oip} suspended, for repeated failed login\.$" }
  120. remove_tag => [ "_grokparsefailure"]
  121. }
  122. } ### dos filter triggered
  123.  
  124. ### overquota warnings
  125. if "LmtpServer" in [message] and "Over quota" in [message] {
  126. mutate { add_tag => ["over_quota"] }
  127. grok {
  128. patterns_dir => ["/etc/logstash/patterns"]
  129. #mailbox.log_14
  130. match => { 'message' => "^\[LmtpServer\-(?<lmtpserver_id>[\d]*)\]\s*\[name=(?<account>[^;]*).*" }
  131. remove_tag => [ "_grokparsefailure"]
  132. }
  133. } ### overquota warnings
  134.  
  135. ### purge operations
  136. if "MailboxPurge" in [message] {
  137. mutate { add_tag => ["mailbox_purge"] }
  138. if "Deleting Message" in [message] {
  139. mutate { add_tag => ["purging_message"] }
  140. grok {
  141. patterns_dir => ["/etc/logstash/patterns"]
  142. #mailbox.log_15
  143. match => { 'message' => "^\[MailboxPurge\] \[name=(?<account>[^;]*)" }
  144. remove_tag => [ "_grokparsefailure"]
  145. }
  146. }
  147. } ### purge operations
  148.  
  149. ### delegated auths
  150. if "DelegateAuth" in [message] and "delegated access" in [message] {
  151. mutate { add_tag => ["delegated_access"] }
  152. grok {
  153. patterns_dir => ["/etc/logstash/patterns"]
  154. #mailbox.log_16
  155. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};port=[\d]*;ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*).*authenticated account=(?<authenticated_account>[^,]*).*target account=(?<target_account>[^;]*)$" }
  156. #mailbox.log_20
  157. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(aname=%{ZIMBRA_USER:aname})?mid=[\d]*;(oip|ip)=%{IP:oip};port=[\d]*;ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*).*authenticated account=(?<authenticated_account>[^,]*).*target account=(?<target_account>[^;]*)$" }
  158. remove_tag => [ "_grokparsefailure"]
  159. }
  160. } ### delegated auths
  161.  
  162. ### spam reports
  163. if "Junk-NotJunk-Handler" in [message] {
  164. mutate { add_tag => ["spam_reporting"] }
  165. grok {
  166. patterns_dir => ["/etc/logstash/patterns"]
  167. #mailbox.log_17
  168. match => { 'message' => "^\[Junk-NotJunk-Handler].*account=%{ZIMBRA_USER:account}.*isSpam=(?<spam>(true|false))" }
  169. remove_tag => [ "_grokparsefailure"]
  170. }
  171. } ### spam reports
  172. } ### mailbox.log
  173.  
  174. ### audit.log
  175. if [log][file][path] == "/opt/zimbra/log/audit.log" {
  176. grok {
  177. match => { 'message' => "^%{TIMESTAMP_ISO8601:logtime} %{LOGLEVEL:loglevel}[[:space:]]*%{GREEDYDATA:message}$" }
  178. overwrite => ["message"]
  179. }
  180.  
  181. date {
  182. match => [ "logtime", "yyyy-MM-dd HH:mm:ss,SSS" ]
  183. target => "@timestamp"
  184. locale => "en_US"
  185. tag_on_failure => ["date_filter_failed"]
  186. }
  187.  
  188. mutate { add_tag => [ "zimbra" ] }
  189.  
  190. grok {
  191. patterns_dir => ["/etc/logstash/patterns"]
  192. add_tag => ["authentication_success"]
  193. #audit.log 1
  194. match => { 'message' => "^%{ZIMBRA_MAILBOX_ID_SOAP} \[name=%{ZIMBRA_USER:account};(oip|ip)=%{IP:oip};ua=%{ZIMBRA_CLIENT:client};soapId=(?<soapid>[^;]*);\] security - cmd=(Auth|AdminAuth); account=%{ZIMBRA_USER}; protocol=[\w]*;$" }
  195. }
  196. } ### audit.log
  197.  
  198. ### zimbra.log
  199. if [log][file][path] == "/var/log/zimbra.log" {
  200. grok {
  201. match => { 'message' => "^%{SYSLOGTIMESTAMP:logtime}\s%{IPORHOST:hostname}[[:space:]]*%{GREEDYDATA:message}$" }
  202. overwrite => ["message"]
  203. }
  204.  
  205. date {
  206. match => [ "logtime", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
  207. target => "@timestamp"
  208. locale => "en_US"
  209. tag_on_failure => ["date_filter_failed"]
  210. }
  211.  
  212. mutate { add_tag => [ "zimbra" ] }
  213.  
  214. grok {
  215. patterns_dir => ["/etc/logstash/patterns"]
  216. add_tag => ["spam"]
  217. #zimbra.log 1
  218. match => { 'message' => "^amavis\[[\d]*\]: \([\d\-]*\) [\w\-]*, <%{EMAILADDRESS:sender}> -> <%{EMAILADDRESS:rcpt}>, (?<isspam>(Yes|No)), score=(?<spamscore>[\d\.]*) required=[\d\.]* tests=\[(?<tests>[^]]*)\] autolearn=(yes|no|disabled) autolearn_force=(yes|no|disabled)$" }
  219. }
  220.  
  221. grok {
  222. patterns_dir => ["/etc/logstash/patterns"]
  223. add_tag => ["virus"]
  224. #zimbra.log 2
  225. match => { 'message' => "^amavis\[[\d]*\]: \([\d\-]*\) (?<virus>Blocked INFECTED) \((?<signature>[\w\._\-]*)\) {DiscardedInbound,Quarantined}, \[%{IPORHOST}\]:[\d]* \[%{IPORHOST:ip}\] <(?<sender>%{EMAILADDRESS})> -> <(?<rcpt>%{EMAILADDRESS})>, quarantine: %{EMAILADDRESS}, Queue-ID: [\w]*, Message-ID: <[^>]*>, mail_id: (?<mailid>[\w]*), Hits: [\w,\-:\s]* ms$" }
  226. }
  227. } ### zimbra.log
  228.  
  229. # general geoip
  230. if ("" in [oip]) {
  231. geoip { source => "oip" }
  232. }
  233. }
  234.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!