import re
import threading
import queue

ParsedQue = queue.Queue()

def openFile():
    file = open('links.txt', 'rb')
    for line in file:
        startParsing(line)


def startParsing(targetLink):
    firstThread = threading.Thread(target=parseFirstPart(targetLink)).start()
    secondThread = threading.Thread(target=parseSecondPart(targetLink)).start()
    thirdThread = threading.Thread(target=parseThirdPart(targetLink)).start()
    forthThread = threading.Thread(target=parseForthPart(targetLink)).start()
    fifthThread = threading.Thread(target=parseFifthPart(targetLink)).start()
    sixThread = threading.Thread(target=parseSixPart(targetLink)).start()
    print('Done : ' + str(targetLink))

## regexp methods
def parseFirstPart(targetLink):
    res = re.findall(r'\.php\?[a-zA-z0-9]{1,30}=' , str(targetLink))
    if not res:
        pass
    else:
        ParsedQue.put(res[0])

def parseSecondPart(targetLink):
    res = re.findall(r'&[a-zA-Z0-9]{0,20}[_]{0,1}[a-zA-Z0-9]{0,20}=' , str(targetLink))
    if res:
        z = res[0].replace('&' , '.php?')
        ParsedQue.put(z)
    else:
        pass

def parseThirdPart(targetLink):
    res = re.findall(r'//[w]{3}\.{0,1}[a-zA-Z0-9]{0,30}' , str(targetLink))
    if res:
        z = res[0].replace('//www.' , '.php?')
        ParsedQue.put(z + '=')
    else:
        pass

def parseForthPart(targetLink):
    res = re.findall(r'//[^w]{3}[a-zA-Z0-9]{0,30}' , str(targetLink))
    if res:
        z = res[0].replace('//' , '.php?')
        ParsedQue.put(z + '=')
    else:
        pass
############################################ This part
def parseFifthPart(targetLink):
    res = re.findall(r'=[a-zA-Z]{1,20}\_[a-zA-Z]{1,20}' , str(targetLink))
    if res:
        z = res[0].replace('=' , '.php?')
        ParsedQue.put(z + '=')
    else:
        pass

def parseSixPart(targetLink):
    res = re.findall(r'=[a-zA-Z]{1,20}' , str(targetLink))
    if res:
        z = res[0].replace('=' , '.php?')
        ParsedQue.put(z + '=')
    else:
        pass
#############################################

##creatin file with result
def fileWritting():
    result = open('result.txt' , 'w')
    getArray = removeDublicates()
    privatArray = checkForExistingDorcs(getArray)
    for row in privatArray:
        result.write(str(row) + '\n')
    result.close()

## delete copies
def removeDublicates():
    result = []
    mass = []
    while not ParsedQue.empty():
        mass.append(ParsedQue.get())

    for row in mass:
        if row not in result:
            result.append(row)
    return result

def checkForExistingDorcs(mass):
    res = []
    for row in mass:
        if row not in open('base.txt').read():
            res.append(row)
            file = open('base.txt' , 'a')
            file.write(row + '\n')
            file.close()
    print('========================================')
    print('Created ' + str(len(res)) + ' new dorks')
    print('========================================')
    return res




openFile()
fileWritting()