API tools faq
paste
EmpireJordan

hacking openvpn

EmpireJordan
Mar 29th, 2017
304
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 8.89 KB | None | 0 0
raw download clone embed print report
  1. # coding=utf-8
  2. ##
  3. #  @file make-antizapret.py
  4. #  @brief Script to generate the list of route commands for OpenVPN server to circumvent Russian internet blacklist
  5. #  It also generates some statistics and lists
  6. #  
  7. import sys, codecs, ctypes, win_unicode_console, requests, pygeoip, geoip2.database, json, csv
  8. from netaddr import IPAddress, IPNetwork, cidr_merge, all_matching_cidrs
  9.  
  10. win_unicode_console.enable()
  11.  
  12. cnt2en = {} #Entries by bureau
  13. cnt2ip = {} #IPs by bureau
  14. cnt2delo = {} #Entries by суд
  15. cnt2deloIP = {} #IPs by суд
  16. domains = {}
  17. #Special lists
  18. cnt = {}
  19. cnt['Total Cloudflare'] = 0
  20. cnt['Total AmazonAWS'] = 0
  21. cnt['Total DO'] = 0
  22. cnt['Total Cloudflare by IP'] = 0
  23. cnt['Total AmazonAWS by IP'] = 0
  24. cnt['Total DO by IP'] = 0
  25.  
  26. #entries
  27. cnt['Total Entries']=0
  28. cnt['IP-only Entries']=0 #IP only
  29. cnt['Domain-only Entries']=0 #Domain name, but no URL
  30. cnt['HTTPS Entries']=0 #URL, but HTTPS
  31. cnt['HTTP Entries']=0 #URL and HTTP
  32. cnt['Other Entries']=0 #Some other protocol
  33. #unique IPs
  34. cnt['Total IPs']=0
  35. cnt['IP-only Unique IPs']=0 #IP only
  36. cnt['Domain-only IPs']=0 #Domain name, but no URL
  37. cnt['HTTPS IPs']=0 #URL, but HTTPS
  38. cnt['HTTP IPs']=0 #URL and HTTP
  39. cnt['Other IPs']=0 #Some other protocol
  40. #Lists of IPs
  41. all=set()
  42. allbyip=set() #Domain-only and IP-only
  43. probablybyip=set() #HTTPS, Domain-only and IP-only
  44. allbydomain=set()
  45. allbyhttps=set()
  46. allbyhttp=set()
  47. allbyother=set()
  48.  
  49. templist=[]
  50. n=0
  51. #Prefixes to filter-merge
  52. prefix=['*.', 'www.', 'm.', 'mobi.', 'mobile.', 'ru.', 'ru.www.' , 'www.ru.', 'wap.', 'pda.', 'en.', 'ua.', 'www0.', 'www1.', 'www2.', 'www3.', 'www4.', 'www5.', 'www6.', 'www7.', 'www8.', 'www9.', 'wwww.']
  53.  
  54. print ('Parsing Blacklist')
  55.  
  56. with open ('D:/Github/z-i/dump.csv', 'r') as csvfile:
  57.     csvr = csv.DictReader (csvfile, fieldnames=['ip','dom','url','who','why','when'], restval='', delimiter=';')
  58.     for row in csvr:
  59.         print ('Processed '+str(csvr.line_num)+' entries.', end='\r', flush=True)
  60.         if row['ip'].find('Updated')>=0 or row['when']=='':
  61.             pass #do nothing
  62.         else:
  63.             cnt['Total Entries']+=1
  64.             who = row['who']
  65.             why = row['why']
  66.             try: # статистика по ведомствам
  67.                 cnt2en[who]+=1
  68.             except KeyError:
  69.                 cnt2en[who] = 1
  70.             try: # списки доменов по ведомствам без префиксов
  71.                 domain=row['dom']
  72.                 for pre in prefix:
  73.                     if domain[0:len(pre)]==pre:
  74.                         domain = domain[len(pre):len(domain)]
  75.                 if domains[who].count(domain)==0:
  76.                     domains[who].append(domain)
  77.             except KeyError:
  78.                 domains[who] = []
  79.                 domains[who].append(domain)
  80.  
  81.             allip = row['ip'].split('|')
  82.            
  83.             if (row['who']=='суд'): #заполнение статистики судебных дел
  84.                 try:
  85.                     cnt2delo[why]+=1
  86.                 except KeyError:
  87.                     cnt2delo[why] = 1
  88.                 for ip in allip:
  89.                     ip2 = ip.strip()
  90.                     k1 = why+' '+ip2
  91.                     if k1 in templist:
  92.                         pass
  93.                     else:
  94.                         templist.append(k1)
  95.                         try:
  96.                             cnt2deloIP[why]+=1
  97.                         except KeyError:
  98.                             cnt2deloIP[why] = 1
  99.                
  100.             for ip in allip:
  101.                 ip2 = ip.strip()
  102.                 if ip2 not in all:
  103.                     all.add(ip2)
  104.                     try: # статистика по ведомствам
  105.                         cnt2ip[who]+=1
  106.                     except KeyError:
  107.                         cnt2ip[who] = 1
  108.             if ((row['url']=='') and (row['dom']=='')) or (row['dom']==row['ip']):
  109.                 cnt['IP-only Entries']+=1
  110.                 for ip in allip:
  111.                     ip2 = ip.strip()
  112.                     if ip2 not in allbyip:
  113.                         allbyip.add(ip2)
  114.             elif (row['url']=='') and (row['dom']!='') and (row['dom']!=row['ip']):
  115.                 cnt['Domain-only Entries']+=1
  116.                 for ip in allip:
  117.                     ip2 = ip.strip()
  118.                     if (ip2 not in allbydomain):
  119.                         allbydomain.add(ip2)
  120.             elif (row['url'].count('https://')>0) and (row['url'].count('http://')==0):
  121.                 cnt['HTTPS Entries']+=1
  122.                 for ip in allip:
  123.                     ip2 = ip.strip()
  124.                     if (ip2 not in allbyhttps):
  125.                         allbyhttps.add(ip2)
  126.             elif row['url'].count('http://')>0:
  127.                 cnt['HTTP Entries']+=1
  128.                 for ip in allip:
  129.                     ip2 = ip.strip()
  130.                     if (ip2 not in allbyhttp):
  131.                         allbyhttp.add(ip2)
  132.             else:
  133.                 cnt['Other Entries']+=1
  134.                 for ip in allip:
  135.                     ip2 = ip.strip()
  136.                     if (ip2 not in allbyother):
  137.                         allbyother.add(ip2)
  138.  
  139. print ('')
  140. print ('Cleanup')
  141.  
  142. for ip in (allbyhttp & (allbyhttps | allbydomain | allbyip | allbyother)):
  143.     allbyhttp.remove(ip)
  144. for ip in (allbyhttps & (allbydomain | allbyip | allbyother)):
  145.     allbyhttps.remove(ip)
  146. for ip in (allbydomain & (allbyip | allbyother)):
  147.     allbydomain.remove(ip)
  148. for ip in (allbyip & allbyother):
  149.     allbyip.remove(ip)
  150.  
  151. probablybyip = allbyhttps | allbydomain | allbyip | allbyother
  152.  
  153. cnt['Total IPs']=len(all)
  154. cnt['HTTP IPs']=len(allbyhttp)
  155. cnt['HTTPS IPs']=len(allbyhttps)
  156. cnt['Domain-only IPs']=len(allbydomain)
  157. cnt['IP-only Unique IPs']=len(allbyip)
  158. cnt['Other IPs']=len(allbyother)
  159.  
  160. #all[] contains all unique IPs for making VPN route table
  161.  
  162. print ('Merging routes')
  163.  
  164. cidrs = cidr_merge (all) # merging adgacent IPs into ranges
  165. f = open ('addrlist.txt', 'w')
  166. n=0
  167. for netw in cidrs:
  168.     n+=1
  169.     print ('Processed '+'{:.1%}'.format(n/(len(cidrs))), end='\r', flush=True)
  170.     route = 'push "route ' + str(netw.ip) + ' ' + str(netw.netmask)+ '"'
  171.     f.write (route + '\n')
  172. f.close()
  173.  
  174. print ('Calculating country statistics')
  175.  
  176. #Creating country stats
  177. countrylist = {}
  178. gi = geoip2.database.Reader('GeoLite2-Country.mmdb')
  179. n=0
  180. for addr in all:
  181.     n+=1
  182.     print ('Processed '+'{:.1%}'.format(n/len(all)), end='\r', flush=True)
  183.     try:
  184.         response = gi.country(addr)
  185.         rcountry = response.country.name
  186.     except geoip2.errors.AddressNotFoundError:
  187.         rcountry = 'None'
  188.     if rcountry is None:
  189.         rcountry = 'None'
  190.     if rcountry in countrylist:
  191.         countrylist[rcountry] += 1
  192.     else:
  193.         countrylist[rcountry] = 1
  194. gi.close()
  195.  
  196. print ('Calculating subnet statistics')
  197.  
  198. # Cloudflare network list. From https://www.cloudflare.com/ips-v4
  199. cloudflaresubnetlist = ['103.21.244.0/22','103.22.200.0/22','103.31.4.0/22','104.16.0.0/12','108.162.192.0/18', '131.0.72.0/22', '141.101.64.0/18', '162.158.0.0/15', '172.64.0.0/13', '173.245.48.0/20', '188.114.96.0/20', '190.93.240.0/20', '197.234.240.0/22', '198.41.128.0/17', '199.27.128.0/21']
  200. digitaloceansubnetlist = cidr_merge(['178.62.0.0/18','178.62.64.0/18','178.62.128.0/18','178.62.192.0/18','138.68.0.0/16','45.55.0.0/16','45.55.108.0/22','45.55.112.0/22','104.131.0.0/16','104.236.0.0/16','107.170.0.0/16','138.197.56.0/22','138.197.60.0/22','138.197.192.0/20','138.197.128.0/20','138.197.144.0/20','159.203.64.0/20','159.203.80.0/20','159.203.96.0/20','159.203.112.0/20','159.203.128.0/20','159.203.144.0/22','159.203.148.0/22','159.203.160.0/20','159.203.192.0/18','159.203.0.0/18','159.203.52.0/22','162.243.0.0/16','192.241.192.0/19','198.199.96.0/20']) #probably not complete
  201. # Generate Amazon networks list
  202. data = requests.get('https://ip-ranges.amazonaws.com/ip-ranges.json')
  203. loaded = json.loads(data.text)['prefixes']
  204. amazonaws = []
  205. for addr in loaded:
  206.     if addr['service']=='AMAZON':
  207.         amazonaws.append(addr['ip_prefix'])
  208. amazonawssubnetlist = cidr_merge(amazonaws)
  209.  
  210. #Count special lists
  211. n=0
  212. for addr in probablybyip:
  213.     n+=1
  214.     print ('Processed '+'{:.1%}'.format(n/(len(all))), end='\r', flush=True)
  215.     if all_matching_cidrs(addr, cloudflaresubnetlist):
  216.         cnt['Total Cloudflare by IP'] += 1
  217.         cnt['Total Cloudflare'] += 1
  218.     elif all_matching_cidrs(addr, amazonawssubnetlist):
  219.         cnt['Total AmazonAWS by IP'] += 1
  220.         cnt['Total AmazonAWS'] += 1
  221.     elif all_matching_cidrs(addr, digitaloceansubnetlist):
  222.         cnt['Total DO by IP'] += 1
  223.         cnt['Total DO'] += 1
  224. for addr in (all-probablybyip):
  225.     n+=1
  226.     print ('Processed '+'{:.1%}'.format(n/(len(all))), end='\r', flush=True)
  227.     if all_matching_cidrs(addr, cloudflaresubnetlist):
  228.         cnt['Total Cloudflare'] += 1
  229.     elif all_matching_cidrs(addr, amazonawssubnetlist):
  230.         cnt['Total AmazonAWS'] += 1
  231.     elif all_matching_cidrs(addr, digitaloceansubnetlist):
  232.         cnt['Total DO'] += 1
  233.  
  234.  
  235. print ('Output                   ')
  236. print('')
  237. #Output
  238. f = open('addrliststats.txt', 'w', encoding='utf-8')
  239. for k in sorted(cnt):
  240.     print ('{:<27} {:>5}'.format(k, cnt[k]))
  241.     f.write('{:<27} {:>5}'.format(k, cnt[k])+'\n')
  242. f.write('\nEntries:\n')
  243. for k in sorted(cnt2en):
  244.     f.write('{:<27} {:>5}'.format(k, cnt2en[k])+'\n')
  245. f.write('\nUnique IPs:\n')
  246. for k in sorted(cnt2ip):
  247.     f.write('{:<27} {:>5}'.format(k, cnt2ip[k])+'\n')
  248. print ('')
  249. f.write('\n')
  250. for k in sorted(countrylist):
  251.     print ('{:<27} {:>5}'.format(k, countrylist[k]))
  252.     f.write('{:<27} {:>5}'.format(k, countrylist[k])+'\n')
  253. f.close()
  254.  
  255. for k in sorted(domains):
  256.     f = open('addrlistdomains-'+ k +'.txt', 'w')
  257.     domains[k].sort()
  258.     for l in domains[k]:
  259.         f.write(l + '\n')
  260.     f.close()
  261.  
  262. f = open('addrlistdomains-by-IP.txt', 'w', encoding='utf-8')
  263. for k in probablybyip:
  264.     f.write(k + '\n')
  265. f.close()
  266.  
  267. f = open('addrlistdomains-dela.csv', 'w', encoding='utf-8')
  268. f.write('Дело, записей, IP\n')
  269. for k in sorted(cnt2delo):
  270.     f.write('\"{:<20}\", {:>5}, {:>5}'.format(k, cnt2delo[k], cnt2deloIP[k])+'\n')
  271. f.close()
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!