API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Nov 1st, 2018
207
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 10.28 KB | None | 0 0
raw download clone embed print report
  1. import os
  2. import sys
  3. import bs4
  4. import csv
  5. from operator import methodcaller
  6. from collections import OrderedDict
  7. import lxml.html
  8. import re
  9. import sys
  10.  
  11. def escstr(s):
  12.     return s.strip().replace("\n", "\\n")\
  13.             .replace("\r", "\\r")\
  14.             .replace('"', '\"')\
  15.             .replace(";", "\;").encode("utf-8")
  16.  
  17. def arr2csvR(arr):
  18.     rc = ""
  19.     for i in arr:
  20.         rc += escstr(i) + ";"
  21.     return rc + '\n' #no need for unterminated rows this time around
  22.  
  23. def textof(elem):
  24. #    while len(elem):
  25. #        elem = elem[0]
  26.     return elem.text_content()
  27.  
  28. def tohdr(s):
  29.     return re.sub(r' +', '_', re.sub(r'[^a-zA-Z0-9]', ' ', s.lower()).strip()).encode("utf-8")
  30.  
  31. physicalcols = {'next': 0}
  32. admissioncols = {'next': 0}
  33. statuscols = {'next': 0}
  34. sentencingcols = {'next' : 0}
  35.  
  36. for fn in sorted(os.listdir("inmates"))[:30000]: #for debug: add [:10000] or some such to make it go faster, note that this might could cause errors in below if it's too low
  37.     first_sentencing = True
  38.     if fn.endswith(".html"):
  39.         r = lxml.html.parse(os.path.join("inmates/", fn)).getroot()
  40.         main = r[1][0][4][0][0][0]
  41.         if len(main) == 1:
  42.             continue #this inmate doesn't exist
  43.         indices = {'physical': None, 'admission': None, 'status': None, 'sentencing': None}
  44.         for i, item in enumerate(main):
  45.             text = item.text_content()
  46.             if len(item) == 1:
  47.                 if text == "PHYSICAL PROFILE": #it's a header
  48.                     indices["physical"] = i+1
  49.                 elif text == "ADMISSION / RELEASE / DISCHARGE INFO":
  50.                     indices["admission"] = i+1
  51.                 elif text == "SENTENCING INFORMATION":
  52.                     indices["sentencing"] = i+1
  53.             if text.startswith("Parent Institution: ") or text.startswith("Alias: "): #maybe elif? can they be 0?
  54.                 indices['status'] = i
  55.         if None in indices.values():
  56.             print "Failed at " + fn
  57.             print indices
  58.             quit(1)
  59.         for i in main[indices["physical"]]:
  60.             if i[0].text_content() not in physicalcols:
  61.                 physicalcols[i[0].text_content()] = physicalcols['next']
  62.                 physicalcols['next'] += 1
  63.         for i in main[indices["admission"]]:
  64.             if i[0].text_content() not in admissioncols:
  65.                 admissioncols[i[0].text_content()] = admissioncols['next']
  66.                 admissioncols['next'] += 1
  67.         for i in main[indices["status"]]:
  68.             if i[0].text_content() not in statuscols and len(i[0].text_content().strip()):
  69.                 statuscols[i[0].text_content()] = statuscols['next']
  70.                 statuscols['next'] += 1
  71.         for i in main[indices['sentencing']]:
  72.             if len(i[0].text_content().strip()):
  73.                 if first_sentencing:
  74.                     if i[0].text_content() not in sentencingcols:
  75.                         sentencingcols[i[0].text_content()] = sentencingcols['next']
  76.                         sentencingcols['next'] += 1
  77.                 else:
  78.                     assert i[0].text_content() in sentencingcols
  79.         first_sentencing = False
  80.  
  81. print physicalcols
  82. print admissioncols
  83. print statuscols
  84. print sentencingcols
  85.  
  86. files = {
  87. 'physical': open("csv/physical.csv", 'w'),
  88. 'admission': open("csv/admission.csv", 'w'),
  89. 'sentencing': open("csv/sentencing.csv", 'w'),
  90. #'warrant': open("csv/warrant.csv", 'w'),
  91. #'warning': open("csv/warning.csv", 'w'),
  92. #'captured': open("csv/captured.csv", 'w'),
  93. 'marks': open("csv/marks.csv", 'w'),
  94. 'status': open("csv/status.csv", 'w'),
  95. }
  96.  
  97. files['marks'].write(arr2csvR(['id', 'mark']))
  98. files['status'].write(arr2csvR(['id'] + map(tohdr, map(lambda i: statuscols.keys()[statuscols.values().index(i)], range(statuscols['next'])))))
  99. files['physical'].write(arr2csvR(['id'] + map(tohdr, map(lambda i: physicalcols.keys()[physicalcols.values().index(i)], range(physicalcols['next'])))))
  100. files['admission'].write(arr2csvR(['id'] + map(tohdr, map(lambda i: admissioncols.keys()[admissioncols.values().index(i)], range(admissioncols['next'])))))
  101. files['sentencing'].write(arr2csvR(['id'] + map(tohdr, map(lambda i: sentencingcols.keys()[sentencingcols.values().index(i)], range(sentencingcols['next'])))))
  102.  
  103. for fn in sorted(os.listdir("inmates")):
  104.     if fn.endswith(".html"):
  105.         r = lxml.html.parse(os.path.join("inmates/", fn)).getroot()
  106.         main = r[1][0][4][0][0][0]
  107.         if len(main) == 1:
  108.             continue #this inmate doesn't exist
  109.         physical = {}
  110.         admission = {}
  111.         id = ''
  112.         armed = False
  113.         captured = False
  114.         indices = {'physical': None, 'admission': None, 'sentencing': None, 'status': None}
  115.         classified = {}
  116.         status = {}
  117.         for i, item in enumerate(main):
  118.             text = item.text_content()
  119.             if len(item) == 1:
  120.                 if text == "PHYSICAL PROFILE": #it's a header
  121.                     indices['physical'] = i+1
  122.                     classified[i] = True
  123.                     classified[i+1] = True
  124.                 elif text == "ADMISSION / RELEASE / DISCHARGE INFO":
  125.                     indices['admission'] = i+1
  126.                     classified[i] = True
  127.                     classified[i+1] = True
  128.                 elif text == "SENTENCING INFORMATION":
  129.                     indices["sentencing"] = i+1
  130.                     classified[i] = True
  131.                     classified[i+1] = True
  132.                 elif re.match("[A-Za-z][0-9]{5} - ", text):
  133.                     id = text[0:6]
  134.                     classified[i] = True
  135.                 elif text.startswith("Warrant Information"):
  136. #                    indices['warrant'] = i
  137.                     classified[i] = True
  138.                 elif text.startswith("Date of Birth: "):
  139.                     "do nothing, we already have physical profile"
  140.                 elif text.startswith("The information"): #disclaimer
  141.                     classified[i] = True
  142.                 elif text.startswith("IDOC TOLL FREE"): #warning
  143.                     classified[i] = True
  144.                 elif text == "Captured":
  145.                     classified[i] = True
  146.                     "do nothing"
  147.                 else:
  148.                     print "unexpected text: " + text
  149.                     quit(2)
  150.             elif text.startswith("Parent Institution: ") or text.startswith("Alias: "):
  151.                 indices['status'] = i
  152.                 classified[i] = True
  153.  
  154.         assert None not in indices.values()
  155.         for i, item in enumerate(main):
  156.             if i not in classified:
  157.                 if len(item):
  158.                     assert len(item) != 1
  159.                     text = item.text_content()
  160.                     if text.startswith(" MARKS, SCARS, & TATTOOS"):
  161.                         for m in item[1:]:
  162.                             files['marks'].write(arr2csvR([id, m.text_content()]))
  163.                     else:
  164.                         print "unexpected text: " + text
  165.                         quit(3)
  166.             elif i in indices.values():
  167.                 type = indices.keys()[indices.values().index(i)]
  168.                 arr = [id]
  169.                 if type == 'sentencing':
  170.                     #9 fields and a spacer, last also has the spacer
  171.                     assert len(item) % 9 == 0
  172.                     for i in range(len(item)/9):
  173.                         files['sentencing'].write(arr2csvR([id] + map(lambda x: x[1].text_content(), item[9*i:9*i+8])))
  174.                 else:
  175.                     c = {'physical': physicalcols,
  176.                         'admission': admissioncols,
  177.                         'status': statuscols}[type]
  178.                     for k in range(c['next']):
  179.                         found = False
  180.                         for m in item:
  181.                             if len(m[0].text_content().strip()) and c[m[0].text_content()] == k:
  182.                                 found = True
  183.                                 if m[0].text_content() == 'Sex Offender Registry Required':
  184.                                     arr += ["true"] #there's no m[1], so we might as well handle it special case
  185.                                 else:
  186.                                     if type == 'physical':
  187.                                         if m[0].text_content() == "Weight:  ":
  188.                                             txt = m[1].text_content()
  189.                                             if txt[-5:] == " lbs.":
  190.                                                 arr += [txt[0:-5]] #"111 lbs." -> 111
  191.                                             elif txt == "Not Available":
  192.                                                 arr += ["N/A"]
  193.                                             else:
  194.                                                 print "unknown weight " + txt
  195.                                                 quit(4)
  196.                                         elif m[0].text_content() == "Height: ":
  197.                                             txt = m[1].text_content()
  198.                                             if re.match(". ft. .. in.", txt):
  199.                                                 foot = int(txt[0])
  200.                                                 inch = int(txt[6:8])
  201.                                                 arr += [str(foot*12+inch)]
  202.                                             elif txt == "Not Available":
  203.                                                 arr += ["N/A"]
  204.                                             else:
  205.                                                 print "unknown weight " + txt
  206.                                                 quit(5)
  207.                                         else:
  208.                                             if not len(m) > 1:
  209.                                                 arr += ['']
  210.                                             else:
  211.                                                 arr += [m[1].text_content()]
  212.                                     else:
  213.                                         if not len(m) > 1:
  214.                                             arr += ['']
  215.                                         else:
  216.                                             arr += [m[1].text_content()]
  217.                         if not found:
  218.                             arr += ['']
  219.                     files[type].write(arr2csvR(arr))
  220.             else:
  221.                 assert i in classified
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!