Guest User

Untitled

a guest
Nov 30th, 2018
126
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.91 KB | None | 0 0
  1. #! /usr/bin/env python
  2.  
  3. import datetime
  4. import difflib
  5. import gzip
  6.  
  7. # Mapping between files and their archived locations
  8. # Must be formatted with a datestamp in the form of `YYmmdd`
  9. ARCHIVE_CONF_FMTS = [
  10. ("flow.conf", "/archive/{0}/eod/flow-eod_{0}.conf.gz"),
  11. ("riskmgr.conf", "/archive/{0}/eod/riskmgr-eod_{0}.conf.gz"),
  12. ("risksettings.conf", "/archive/{0}/eod/risksettings-eod_{0}.conf.gz"),
  13. ]
  14.  
  15. # Returns true if the given date is a weekday (Mon-Fri)
  16. def is_weekday(date):
  17. return 0 <= date.weekday() <= 4
  18.  
  19. # Gets a file's path from the archive from a specific date
  20. def get_archive_path(fname, date):
  21. path = next(p for f, p in ARCHIVE_CONF_FMTS if f == fname)
  22. return path.format(date.strftime("%Y%m%d"))
  23.  
  24. # Read a config from the archive, returns the list of lines
  25. def read_archived_config(fname, date):
  26. fpath = get_archive_path(fname, date)
  27. fhandle = gzip.open(fpath)
  28. flines = fhandle.readlines()
  29. fhandle.close()
  30. return flines
  31.  
  32. # Given a date, returns the date of the following weekday
  33. def next_weekday(date):
  34. weekday = date.weekday()
  35. if weekday == 4: # Friday
  36. return date + datetime.timedelta(days=3)
  37. if weekday == 5: # Saturday
  38. return date + datetime.timedelta(days=2)
  39. return date + datetime.timedelta(days=1)
  40.  
  41. # Given a date, returns the date of the previous weekday
  42. def prev_weekday(date):
  43. weekday = date.weekday()
  44. if weekday == 0: # Monday
  45. return date - datetime.timedelta(days=3)
  46. if weekday == 6: # Sunday
  47. return date - datetime.timedelta(days=2)
  48. return date - datetime.timedelta(days=1)
  49.  
  50. # Opens regular text or gzip file, returning None if failed
  51. def open_config_file(fpath):
  52. try:
  53. if fpath.endswith(".gz"):
  54. return gzip.open(fpath, 'r')
  55. else:
  56. return open(fpath, 'r')
  57. except IOError:
  58. return None
  59.  
  60. # Returns the set of flows from a config file
  61. def read_flows(fpath):
  62. fconf = open_config_file(fpath)
  63. if fconf is None:
  64. return []
  65. flows = [line for line in fconf.readlines() if line.startswith('flow')]
  66. fconf.close()
  67. return flows
  68.  
  69. # Returns the path to the config file archived on that date
  70. def get_config_path(fname, date):
  71. if date == datetime.date.today():
  72. return "/opt/hpr/config/{0}.conf".format(fname)
  73.  
  74. new_name = "{0}-eod_{1}.conf.gz".format(fname, date.strftime("%Y%m%d"))
  75. return "/archive/{0}/eod/{1}".format(date.strftime("%Y%m%d"), new_name)
  76.  
  77. # Returns the flows from a given date, going forward if none are found
  78. def open_next_config(fnames, date):
  79. # Ensure the date used is a weekday
  80. if not is_weekday(date):
  81. date = next_weekday(date)
  82.  
  83. # Open the config files
  84. fpaths = [get_config_path(fname, date) for fname in fnames]
  85. all_flows = [read_flows(fpath) for fpath in fpaths]
  86. filtered_flows = [flows for flows in all_flows if len(flows) != 0]
  87.  
  88. # Either it found no flows today, or found more than one file with flows
  89. if len(filtered_flows) != 1:
  90. return open_next_config(fnames, next_weekday(date))
  91.  
  92. return (date, filtered_flows[0])
  93.  
  94. # Returns the flows from a given date, going forward if none are found
  95. def open_prev_config(fnames, date):
  96. # Ensure the date used is a weekday
  97. if not is_weekday(date):
  98. date = prev_weekday(date)
  99.  
  100. # Open the config files
  101. fpaths = [get_config_path(fname, date) for fname in fnames]
  102. all_flows = [read_flows(fpath) for fpath in fpaths]
  103. filtered_flows = [flows for flows in all_flows if len(flows) != 0]
  104.  
  105. # Either it found no flows today, or found more than one file with flows
  106. if len(filtered_flows) != 1:
  107. return open_prev_config(fnames, prev_weekday(date))
  108.  
  109. return (date, filtered_flows[0])
  110.  
  111. # Opens the files from a given date and a month prior
  112. def open_both_configs(fnames, date):
  113. pdate = date - datetime.timedelta(days=30)
  114.  
  115. if date == datetime.date.today():
  116. # Not running manually; Iterate backwards
  117. return (open_prev_config(fnames, date), open_prev_config(fnames, pdate))
  118. else:
  119. # Running manually; Iterate forwards
  120. return (open_next_config(fnames, date), open_prev_config(fnames, pdate))
  121.  
  122. # Sanitize and format a flow command for presenting to client.
  123. def format_flow(flow):
  124. words = flow.split()[::-1]
  125. sess_id = words[17]
  126. sess_acc = words[16]
  127. sess_sys = words[15]
  128. sess_mic = words[12]
  129. sess_ip = words[6]
  130. sess_port = words[5]
  131. sess_user = words[4]
  132. sess_pass = words[0]
  133.  
  134. sess_socket = "{0}:{1}".format(sess_ip, sess_port)
  135. sess_flow = "Session: {0}".format(sess_id)
  136. sess_user = "Username: {0}".format(sess_user)
  137. sess_comp = "CompID: {0}".format(sess_pass)
  138.  
  139. return (sess_id, "{0}\t{1}\t{2}\t{3}\t{4}\t{5}".format(sess_socket, sess_acc, sess_mic, sess_flow, sess_user, sess_comp))
  140.  
  141. def main(argv):
  142. date = sanitize_args(argv)
  143. conf_names = ["flow", "riskmgr"]
  144. try:
  145. ((cur_date, cur_flows), (old_date, old_flows)) = open_both_configs(conf_names, date)
  146. except RuntimeError as re:
  147. print("Could not find files in archive")
  148. sys.exit(-1)
  149. except IOError as io:
  150. print("Could not open files: {0}".format(io))
  151. sys.exit(-1)
  152.  
  153. print("Comparing flows between dates {0} and {1}".format(cur_date, old_date))
  154. print("\n")
  155. print("Number of flows on {0}: {1}".format(cur_date, len(cur_flows)))
  156. print("Number of flows on {0}: {1}".format(old_date, len(old_flows)))
  157. print("\n")
  158.  
  159. d = difflib.Differ()
  160. diff = list(d.compare(cur_flows, old_flows))
  161.  
  162. possibly_added = [format_flow(line) for line in diff if line.startswith('+')]
  163. possibly_removed = [format_flow(line) for line in diff if line.startswith('-')]
  164.  
  165. possibly_added_ids = [x for (x, flow) in possibly_added]
  166. possibly_removed_ids = [x for (x, flow) in possibly_removed]
  167.  
  168. actually_added = [flow for (sess_id, flow) in possibly_added if not sess_id in possibly_removed_ids]
  169. actually_removed = [flow for (sess_id, flow) in possibly_removed if not sess_id in possibly_added_ids]
  170.  
  171. print("Flows removed: ({0})".format(len(actually_removed)))
  172. print("-" * 48)
  173. for line in actually_removed:
  174. print(line)
  175.  
  176. # Separate by newline
  177. print('\n')
  178.  
  179. print("Flows added: ({0})".format(len(actually_added)))
  180. print("-" * 48)
  181. for line in actually_added:
  182. print(line)
  183.  
  184. # Sanitize the CLI arguments
  185. def sanitize_args(argv):
  186. # If no optional date is provided, use today
  187. if len(argv) < 2:
  188. date = datetime.date.today()
  189.  
  190. # But make sure it's a Tuesday
  191. if date.weekday() != 1:
  192. print("Cannot run script automatically on non-Tuesdays")
  193. print("Supply a date to run manually")
  194. print("")
  195. print_usage_and_exit(argv[0])
  196. else:
  197. date = datetime.datetime.strptime(argv[1], '%Y%m%d').date()
  198.  
  199. return date
  200.  
  201. def print_usage_and_exit(pname):
  202. print("Usage: {0} [YYYYMMDD]".format(pname))
  203. sys.exit(-1)
  204.  
  205. if __name__ == '__main__':
  206. import sys
  207. main(sys.argv)
Add Comment
Please, Sign In to add comment