ScottHelme

Custom Logs

Apr 16th, 2016
413
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. nginx.conf
  2. http {
  3. log_format scott_custom '$ssl_protocol $ssl_cipher $request';
  4. }
  5.  
  6. scotthelme.co.uk.conf
  7. server {
  8. access_log /var/log/nginx/scott_custom.log scott_custom;
  9. }
  10.  
  11. mkdir ~/logs/
  12. cd /var/log/nginx
  13. sudo cp scott_custom.log.*.gz ~/logs/
  14. cd ~/logs/
  15. gunzip *.gz
  16. cat * >> big.log
  17.  
  18. scott@scotthelme:~/logs$ awk '{print $1}' big.log | sort | uniq -c | sort -rn
  19. 6140215 TLSv1.2
  20. 466836 TLSv1
  21. 24787 TLSv1.1
  22. 610 -
  23.  
  24. scott@scotthelme:~/logs$ awk '{print $2}' big.log | sort | uniq -c | sort -rn
  25. 4083598 ECDHE-RSA-AES128-GCM-SHA256
  26. 1567382 ECDHE-RSA-AES256-GCM-SHA384
  27. 533093 ECDHE-RSA-AES256-SHA
  28. 205345 DHE-RSA-AES256-GCM-SHA384
  29. 130156 ECDHE-RSA-AES256-SHA384
  30. 51609 AES256-SHA
  31. 35278 DHE-RSA-AES256-SHA
  32. 10837 ECDHE-RSA-AES128-SHA
  33. 3059 DHE-RSA-AES128-GCM-SHA256
  34. 2977 DHE-RSA-AES256-SHA256
  35. 2512 AES128-GCM-SHA256
  36. 1838 AES256-SHA256
  37. 1783 DES-CBC3-SHA
  38. 1295 DHE-RSA-AES128-SHA
  39. 610 -
  40. 525 AES256-GCM-SHA384
  41. 356 (NONE)
  42. 115 ECDHE-RSA-AES128-SHA256
  43. 69 AES128-SHA
  44. 11 EDH-RSA-DES-CBC3-SHA
  45.  
  46. scott@scotthelme:~/logs$ awk '{print $3}' big.log | sort | uniq -c | sort -rn
  47. 6498247 GET
  48. 76709 HEAD
  49. 50750 POST
  50. 5184 PUT
  51. 427 OPTIONS
  52. 425 PROPFIND
  53. 309 \x15\x03\x01\x00\x02\x01\x00
  54. 140
  55. 61 \x15\x03\x02\x00\x02\x01\x00
  56. 53 \x15\x03\x03\x00\x02\x01\x00
  57. 23 DELETE
  58. 18 CONNECT
  59. 17 Accept-Encoding:
  60. 14 quit
  61. 12 -
  62. 9 \x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01
  63. 7 x
  64. 6 q
  65. 4 \x80\x03\x00\x04\x01\x00\x00\x14\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00d\x00\x00\x00\x07\x00\x01\x00\x00\x80\x03\x00\x01\x00\x00\x01r\x00\x00\x00\x01\x00\x00\x00\x00
  66. 4 \x80\x03\x00\x04\x01\x00\x00\x14\x00\x00\x00\x02\x00\x00\x00\x04\x00\x00\x00d\x00\x00\x00\x07\x00\x01\x00\x00\x80\x03\x00\x01\x00\x00\x01\x5C\x00\x00\x00\x01\x00\x00\x00\x00
  67. 4 \x15\x03\x03\x00\x02\x02(
  68. 3 \x15\x03\x01\x00\x02\x02*
  69. 2 atvv\x0Cvwv\x00tt
  70. 2 TRACE
  71. 1 ~\xF1\xFC\xCA\x81\x0E\x035\xE2\xF9\x95\xFDF(;\xEF\xB7.|\x07\x8A\xD4R\xA9\xD8\xFC\xC9r\x0C\xE2On\xD5\xB8fi$T\x12Z\x86-\x88Y\x8E\xA4\x122\xA5$\xD6B\xA3\x9El\x11o_\xFA<0\xAE\x98X
  72. 1 q\x03\xB3c&68\xA3p\xAC\xC9\x1A\x82\xB6\xAF\x8A!W\xD6\xD2\xF0B\xB6\x1A\xA9\x87B\x0Eg\x98\xF5\xBEX\xB1\xCC\x9B-\xF1\xBDw\xC4\xA8T\xFF\x10\xCA\x9C\xC3\xAD\x82|@\x93X\x1D\xF5\xC0\xB1Y\xDC\xA1{1\x15|0\x9A\xBB\x04<\xB1]S\xF9i\xEB
  73. 1 j
  74. 1 atww\xEEvww\xE2twJ\xF1N\xE5\xA0\xFDUanY\xDB\xF88\xE6\xC3\xEF1;\xAA\x1F\xF2T\xA6v:\xF1\xAE\x1A\xCA\x178\xF0ww\x19\xB7c\xB7}wNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7x\xB7rwBw\xF3\xB7d\xB7~wDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7y\xB7swXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vw
  75. 1 atww\xEEvww\xE2tw7\xE20\xF3\xB5\x15?]\xF9\x0BOa\xDF@\xD2\xE0\x01\xCA\xA9\x95X\xCC\xAE\xE2A\xE6\xE1\xE5\xE7b\x11Uww\x19\xB7c\xB7}wNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7x\xB7rwBw\xF3\xB7d\xB7~wDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7y\xB7swXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vw
  76. 1 atvw\x92vww\x96tv\xDB(\x92\x8FD(=\xDCC\xE2K\xE1\x84\x83\xA4\xC4X\xD9\xD6\xD3\xF1\xC6\x88\xB5\x9C\x96\x12\xD2s\x84\xFF\xC5ww\x19\xB7c\xB7}wNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7x\xB7rwBw\xF3\xB7d\xB7~wDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7y\xB7swXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vww=wwwbwdwwg\x04\x14\x18\x03\x03\x1F\x12\x1B\x1A\x12Y\x14\x18Y\x02\x1Cw|wstwvuw}wkwmw`wnwkwlwowmwawywzw|w{w~w}wTwwwxwvv
  77. 1 atvw\x92vww\x96tvT\xD5s\xFA\x11r\xDDY\x8D\xBA
  78. 1 atvv&vwv:tt\xB1\xD2<\x5C\xDA0\x85\x83'\xBB)\xFA\xDC\xC2\x83\xE2\x9A'\x94\x1Fg8\x91\x1F\xAF\xCC\x9AZ\x1B\x9C\x87#ww\xC1\xB7G\xB7[\xB7_\xB7S\xB7c\xB7}w\xD2w\xD4w\xD6w\xE8w\x1Cw\x1Dw\x1Ew\x1FwNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7E\xB7Y\xB7]\xB7Q\xB7x\xB7rw\xEAwJwBw\xF3\xB7X\xB7\x5C\xB7P\xB7T\xB7d\xB7~w\xD3w\xD5w\xD7w\xE9w\x10w7wHwIwDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7F\xB7Z\xB7^\xB7R\xB7y\xB7sw\xEBwKwXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vww\x19wwwbwdwwg\x04\x14\x18\x03\x03\x1F\x12\x1B\x1A\x12Y\x14\x18Y\x02\x1Cw|wstwvuw}wkwmw`wnwkwlwowmwawywzw|w{w~w}wTwwwzwWwiqvquqtrvrurtsvsusttvtuttuvuuutwxwvv
  79. 1 atvv&vwv:tt\x03\xAA\x1C4/\xEE\xD4\x9B\xEDI\x07\x09\xB8/\x83)%\xBD=p\xDB\xE8\xD7\xC9\xBA~iM**v\xEAww\xC1\xB7G\xB7[\xB7_\xB7S\xB7c\xB7}w\xD2w\xD4w\xD6w\xE8w\x1Cw\x1Dw\x1Ew\x1FwNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7E\xB7Y\xB7]\xB7Q\xB7x\xB7rw\xEAwJwBw\xF3\xB7X\xB7\x5C\xB7P\xB7T\xB7d\xB7~w\xD3w\xD5w\xD7w\xE9w\x10w7wHwIwDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7F\xB7Z\xB7^\xB7R\xB7y\xB7sw\xEBwKwXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vww\x19wwwbwdwwg\x04\x14\x18\x03\x03\x1F\x12\x1B\x1A\x12Y\x14\x18Y\x02\x1Cw|wstwvuw}wkwmw`wnwkwlwowmwawywzw|w{w~w}wTwwwzwWwiqvquqtrvrurtsvsusttvtuttuvuuutwxwvv
  80. 1 atvv&vwv:tt4c\x0C\x16\x22\xF6CjUL\x99r\xE0w\x83\xE3\xDC\xCA-\xF6\xB5\xE72,W\xBF\xA7\xBD\xA9\x0B\xF7\xC3ww\xC1\xB7G\xB7[\xB7_\xB7S\xB7c\xB7}w\xD2w\xD4w\xD6w\xE8w\x1Cw\x1Dw\x1Ew\x1FwNwOw@wAw\xFFw\xF0w\xF1w\xF2\xB7E\xB7Y\xB7]\xB7Q\xB7x\xB7rw\xEAwJwBw\xF3\xB7X\xB7\x5C\xB7P\xB7T\xB7d\xB7~w\xD3w\xD5w\xD7w\xE9w\x10w7wHwIwDwEwFwGw\xEDw\xEEw\xEFw\xE0w2w3w4w5\xB7F\xB7Z\xB7^\xB7R\xB7y\xB7sw\xEBwKwXw\xE1w6wp\xB7f\xB7p\xB7{\xB7uwrws\xB7e\xB7\x7Fwawdwgwz\xB7z\xB7tw}wbwewxw{w~w\x88vww\x19wwwbwdwwg\x04\x14\x18\x03\x03\x1F\x12\x1B\x1A\x12Y\x14\x18Y\x02\x1Cw|wstwvuw}wkwmw`wnwkwlwowmwawywzw|w{w~w}wTwwwzwWwiqvquqtrvrurtsvsusttvtuttuvuuutwxwvv
  81. 1 \x15\x03\x03\x00\x1A\x00\x00\x00\x00\x00\x00\x00\x02\x8B|#Y\xFC\x95y=Z\x22\x22\x1AI#Av\xC6\xA9
  82. 1 \x05\x02\x00\x02
  83. 1 \x04\x01\x1F\x00\x00\x00\x00\x00\x00
  84. 1 \x00\x9C\x00\x01\x1A+<M\x00\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00
  85. 1 TRACK
  86. 1 SSH-2.0-LYGhost_1.2.7-20100630
  87. 1 Hello
  88. 1 FLURP
  89.  
  90. scott@scotthelme:~/logs$ awk '{print $4}' big.log | sort | uniq -c | sort -rn
  91. 662145 /rss/
  92. 209532 /assets/css/Aio.min.css
  93. 205015 /assets/css/rrssb.min.css
  94. 202162 /assets/js/ga.min.js
  95. 195522 /assets/js/rrssb.min.js
  96. 194582 /assets/images/favicon.ico
  97. 160176 /assets/fonts/fontawesome-webfont.woff?v=4.0.3
  98. 137152 /
  99. 133411 /feed/
  100. 105270 /assets/js/disqus-post.min.js
  101. 103310 /rss
  102. 101015 /assets/js/Aio-lib.min.js
  103. 91287 /assets/js/nr.min.js
  104. 79905 /setting-up-le/
  105. 77278 /setting-up-hsts-in-nginx/
  106. 42766 /hardening-your-http-response-headers/
  107. 42697 /assets/js/ads.min.js
  108. 29939 /robots.txt
  109. 27012 /content/images/2015/02/iis-url-rewrite-view-server-variables-back-to-rules.png
  110. 26837 /content/images/2015/02/iis-response-headers.png
  111. 26523 /content/images/2015/02/iis-url-rewrite-view-server-variables.png
  112. 26510 /content/images/2015/02/iis-url-rewrite-add-rule-content-x-powered-by.png
  113. 26509 /content/images/2015/02/iis-url-rewrite-add-rule-content-value.png
  114. 26470 /content/images/2015/02/iis-remove-x-powered-by.png
  115. 26466 /content/images/2015/02/iis-url-rewrite-add-rule-content.png
  116. 26463 /content/images/2015/02/iis-url-rewrite.png
  117. 26459 /content/images/2015/02/iis-url-rewrite-view-server-variables-add.png
  118. 26452 /content/images/2015/02/iis-url-rewrite-add-rules.png
  119. 26451 /content/images/2015/02/iis-response-headers-1.png
  120. 26424 /content/images/2015/02/iis-url-rewrite-add-variable-x-powered-by.png
  121. 26389 /content/images/2015/02/iis-xfo-header.png
  122. 26375 /content/images/2015/02/nginx-server-header-source-modified.png
  123. 26365 /content/images/2015/02/iis-url-rewrite-server-header-with-value.png
  124. 26364 /content/images/2015/02/nginx-server-header-source.png
  125. 26362 /content/images/2015/02/iis-xxss-header.png
  126. 26356 /content/images/2015/02/iis-url-rewrite-blank-server-header.png
  127. 26350 /content/images/2015/02/iis-url-rewrite-add-rules-blank.png
  128. 26336 /content/images/2015/02/iis-server-header.png
  129. 26330 /content/images/2015/02/iis-url-rewrite-view-server-variables-add-value.png
  130. 26323 /content/images/2015/02/nginx-server-header-1.png
  131. 26320 /content/images/2015/02/iis-hsts-header.png
  132. 26313 /content/images/2015/02/iis-csp-header.png
  133. 26305 /content/images/2015/02/iis-xcto-header.png
  134. 26303 /content/images/2015/03/iis-hpkp-header.png
  135. 23119 /content-security-policy-an-introduction/
  136. 22358 /wp-content/uploads/2013/08/pineapple.png
  137. 20161 /still-think-you-dont-need-https/
  138. 19880 /assets/js/disqus.min.js
  139. 19716 /ee-brightbox-router-hacked/
  140. 19441 /hpkp-http-public-key-pinning/
  141. *snip - way too big*
  142.  
  143. scott@scotthelme:~/logs$ awk '{print $5}' big.log | sort | uniq -c | sort -rn
  144. 4167611 HTTP/2.0
  145. 2418259 HTTP/1.1
  146. 45907 HTTP/1.0
  147. 668
  148. 1 RTSP/1.0
  149. 1 :-D
  150. 1 2{A\x11\xC56\xA3\x16v\x8F\x8F\xBB\xE6\x8B\x05k\xF0|\xC3T|dWww\xBD\xB7G\xB7[\xB7_\xB7S\xB7c\xB7}\xB7U\xB7Vw\xD4w\xE8w\x1Cw\x1DwNwOw\xFFw\xF0\xB7n\xB7Ww\xD0w\x1AwMw\xFE\xB7E\xB7Y\xB7]\xB7Q\xB7x\xB7rw\xEAwJwBw\xF3\xB7e\xB7\x7F\xB7k\xB7lwawd\xB7`\xB7mwl\xB7z\xB7tw}\xB7X\xB7\x5C\xB7P\xB7T\xB7d\xB7~\xB7h\xB7iw\xD5w\xE9w\x10w7wDwEw\xEDw\xEEw2w3\xB7o\xB7jw\xD1w\x1BwCw\xECw1\xB7F\xB7Z\xB7^\xB7R\xB7y\xB7sw\xEBwKwXw\xE1w6wp\xB7f\xB7p\xB7awo\xB7{\xB7uwrwswbwewmw~wcwfwnw\x7Fwqw`wtw\x88vww\xF3wwwbwdwwg\x04\x14\x18\x03\x03\x1F\x12\x1B\x1A\x12Y\x14\x18Y\x02\x1Cw|wstwvuw}wCwEwywzwnw|w{wow~w}waw`w\x7FwqwpwcwbwswrwewdwvwuwtwxwgwfwzwUwWqvquqtrvrurtsvsusttvtuttuvuuutvvwxwvv
RAW Paste Data Copied