Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/python
- import re,os,urllib,urllib2,sys,csv,time
- def dedup():
- import os,time
- today_var=time.strftime("%Y%m%d")
- for file in os.listdir('/home/cdrlookup/fraud_calls/cleanlog/'):
- file_str=''
- file_str+=str(file)
- os.system('sort -r --field-separator=, --key=2,3 --unique < /home/cdrlookup/fraud_calls/cleanlog/'+file_str+' --output=/home/cdrlookup/fraud_calls/cleanlog/'+file_str+'_dedup')
- with open('/home/cdrlookup/fraud_calls/cleanlog/'+file_str+'_dedup') as f:
- next(f)
- for line in (f):
- line =line.strip()
- data=line.split(",")
- try:
- #report_time = data[0]
- caller = data[1]
- called = data[2]
- date = data[3]
- time = data[4]
- line=(caller +',' + called + ',' + date +','+ time)
- print line
- except IndexError:
- continue
- #line=(caller +',' + called + ',' + date +','+ time)
- #print line
- def updtcommas():
- today_var=time.strftime("%Y%m%d")
- working_dic = '/home/cdrlookup/fraud_calls/processed_dir/'
- for file in os.listdir('/home/cdrlookup/fraud_calls/callslog/'):
- file_str='/home/cdrlookup/fraud_calls/callslog/'
- file_str+=str(file)
- #print file_str
- with open ('/home/cdrlookup/fraud_calls/processed_dir/'+file+'_processed',"w") as processed_file:
- #processed_file = open('/home/cdrlookup/fraud_calls/processed_dir/'+file_str+'_processed_file','w')
- with open ('/home/cdrlookup/fraud_calls/callslog/'+file) as f:
- #with open(file_str) as f:
- #print processed_file#file_str
- while True:
- c = f.read(1)
- if not c:
- # print ("End of file")
- break
- # print ("Read a character:", c)
- if c=='"':
- processed_file.write(c)
- c = f.read(1)
- while c != '"':
- if c== ',':
- c= ';'
- processed_file.write(c)
- c = f.read(1)
- processed_file.write(c)
- continue
- #processed_file.close()
- def getSPID(input):
- url = 'http://pears.web.att.com/PEARS/services/getPortableData'
- values = {'method' : 'searchByMDN','phonenumbers' : input}
- try:
- data = urllib.urlencode(values)
- req = urllib2.Request(url, data)
- response = urllib2.urlopen(req)
- salida=response.read().replace('\r\n','').replace('\t','')
- lrn=salida.split("'LRNs','width=400px,height=200px,resize=1,center=1'); return false")[1].split("</a>")[0][2:]
- spid=salida.split('<td align="middle">')
- area=spid[-1].split('"display:none">')[1].split('</td>')[0]
- spid=[m for m in spid if m.find(' </td> ')!=-1][0].split('</td>')[0].replace(' ',' ').replace(' ',' ').replace(' ',' ')
- m=salida.split('<td align="middle">')
- geocode=[p for p in m if p.find('</td> ')>-1 and len(p.split('</td>')[0])==3][1][0:3]
- return spid
- except IndexError:
- pass
- def main():
- data = []
- new_line=0
- header =("report_time,caller,called,date,time,ampm,timezone,call_type,goal,ctanumber,url,comments,captcha,client_ip,remote_client_ip,Carrier")
- start_regex=re.compile(r'[^"]?\d{2}/\d{2}/\d{2}[?"]?')
- regex=re.compile(r'^(\d{10})(?:\s|$)')
- regex1=re.compile(r'^(\d)\1{5,}')
- # calling updatecommas method to update every comma in files...
- #file_updt=updtcommas()
- #today_var=time.strftime("%Y%m%d")
- #for file in os.listdir('/home/cdrlookup/fraud_calls/processed_dir/'):
- for file in os.listdir('/home/cdrlookup/fraud_calls/callslog/'):
- file_str=''
- file_str+=str(file)
- #ROBO = open('/home/cdrlookup/fraud_calls/processed_dir/'+file_str)
- ROBO = open('/home/cdrlookup/fraud_calls/callslog/'+file_str)
- #ROBO = open('/home/cdrlookup/fraud_calls/calls_'+today_var+'.log')
- clean_file=open('/home/cdrlookup/fraud_calls/cleanlog/'+file_str+'_clean','w')
- bad_file=open('/home/cdrlookup/fraud_calls/badlog/'+file_str+'_bad','w')
- next(ROBO)
- clean_file.write(header+ "\n")
- for line in (ROBO):
- line =line.strip()
- data=line.split(",")
- try:
- report_time = data[0]
- caller = data[1]
- called = data[2]
- date = data[3]
- time = data[4]
- ampm = data[5]
- timezone = data[6]
- call_type = data[7]
- goal = data[8]
- ctanumber = data[9]
- url = data[10]
- comments = data[11]
- captcha = data[12]
- client_ip = data[13]
- remote_client_ip =data[14]
- except IndexError:
- continue
- tn_caller=re.sub("[^0-9]","",caller)
- tn_caller=tn_caller.lstrip("+")
- tn_caller=tn_caller.lstrip("0")
- tn_caller=tn_caller.lstrip("1")
- tn_called=re.sub("[^0-9]","",called)
- tn_called=tn_called.lstrip("+")
- tn_called=tn_called.lstrip("0")
- tn_called=tn_called.lstrip("1")
- mo1 =(regex.search(tn_caller))
- mo2=(regex.search(tn_called))
- mo3=(regex1.search(tn_caller))
- mo4=(regex1.search(tn_called))
- new_line =(report_time+','+caller+','+called+','+date+','+time+','+ampm+','+timezone+','+call_type+','+goal+','+ctanumber+','+url+','+comments+','+captcha+','+client_ip+','+remote_client_ip)
- # Check if the line started correctely with report_time
- if not start_regex.search(report_time):
- bad_file.write((new_line) + "\n")
- # check if it's 10 digits for the caller and called numbers
- elif not(mo1 and mo2):
- bad_file.write((new_line) + "\n")
- #and not more than 5 times repeated for both caller and called numbers
- elif (mo3 and mo4):
- bad_file.write((new_line) + "\n")
- else:
- carr=getSPID(tn_caller)
- if carr is None:
- clean_file.write((new_line) + ",\"Unknown Carrier\"\n")
- else:
- clean_file.write((new_line) + ",\"" + carr + "\"\n")
- continue
- bad_file.close()
- ROBO.close()
- clean_file.close()
- updtcommas()
- main()
- dedup()
Add Comment
Please, Sign In to add comment
