#!/usr/bin/env python -W ignore::DeprecationWarning

import pymongo
import json
from collections import Counter
from operator import itemgetter

def init_thesis(collection):
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = myclient["Thesis"]
    mycol = mydb[collection]
    return mycol

def init_database(collection):
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = myclient["ChromeExtension"]
    mycol = mydb[collection]
    return mycol

mycol = init_database("API")

# get behavior form file api.json define
# Return json behavior
def GetBehaviorMalicious(behavior):
    with open(r"G:\New\Extensions\KhoaLuan\source\sandbox\api.json") as f:
        _behavior = json.load(f)
    return _behavior[behavior]

def GetApiCalledByExtension(idx):
    list_api_from_database = mycol.find({"extensionId": idx})
    return list_api_from_database

def UninstallBehaviorTracking(api_of_extension):
    _behavior_info = GetBehaviorMalicious("uninstall_other_extension")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    #Checking
    if(api_of_extension in list_api_behavior):
        return True
    return False

def PreventsUninstallTracking(api_of_extension):
    _behavior_info = GetBehaviorMalicious("prevents_extension_uninstall")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    for api_behavior in list_api_behavior:
        list_name_api_of_behavior.append(api_behavior["apiCall"])

    if(api_of_extension["apiCall"] in list_name_api_of_behavior):
        if("argUrl" in api_of_extension.keys() and api_of_extension["argUrl"] in "chrome://extensions/"):
            return True
    return False

def KeyloggerTracking(api_of_extension):
    # Kiem tra apiCall co nam trong danh sach api hanh vi cua keylloging hay khong, cu the la:blinkAddEventListener
    # Neu có api blinkAddEventListener thi kiem tra args duoc truyen vao
    # args [ "#document", "keypress"] hoac "args": [ "#document", "keydown" ] thi return True
    # -> Extension chua hanh vi cua keylogger
    _behavior_info = GetBehaviorMalicious("keylogging_functionality")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    list_args = []
    for api_behavior in list_api_behavior:
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])
        list_args.append(api_behavior["args"])


    if(api_of_extension["apiCall"] in list_name_api_of_behavior):
        if(json.loads(api_of_extension["args"]) in list_args):
            return True
    return False

def StealInformationFormTracking(api_of_extension):
    # Kiem tra blinkAddEventListener api co gia tri args ["FORM","submit"]
    # Neu co thi lay pageUrl
    # Kiem tra pageUrl co activityType la content_script
    # Neu co thi extension da inject script vao page de get form thong tin
    # Den day kiem tra xem pageUrl co api blinkAddEventListener voi tham so ["XMLHttpRequest","load"]
    # Neu co kha nang cao se gui thong tin dang nhap ra ngoai

    _behavior_info = GetBehaviorMalicious("steal_information_form")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    list_args = []
    list_activityType = []
    for api_behavior in list_api_behavior:
        # Kiem tra api trong list_name_api_of_behavior hay khong chua, khong add cac api null
        # Neu chua co thi them vao list
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])
        list_args.append(api_behavior["args"])
        if("activityType" in api_behavior):
            list_activityType.append(api_behavior["activityType"])

    #Kiem tra cac behavior
    if(api_of_extension["apiCall"] in list_name_api_of_behavior):
        if(api_of_extension["args"] in "[\"FORM\",\"submit\"]"):
            find_activity = mycol.find({"extensionId": api_of_extension["extensionId"],"pageUrl":api_of_extension["pageUrl"],"activityType":"content_script"})
            if(len(list(find_activity)) != 0):
                return True
    return False

def BlockAntiVirusSiteTracking(api_of_extension):
    # Kiem tra api co phai Apicall co phai la webRequestInternal.addEventListener
    # Neu la api do thi kiem tra args
    # Args chua hanh dong blocking thi kieu tra tham so domain
    # Neu tham domain co chua cac domain antivius thi return True

    _behavior_info = GetBehaviorMalicious("block_antivirus_site")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    list_args = []
    for api_behavior in list_api_behavior:
        # Kiem tra api trong list_name_api_of_behavior hay khong chua, khong add cac api null
        # Neu chua co thi them vao list
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])
        list_args = (api_behavior["args"])

    #Kiem tra behavior
   # matches = [x for x in white_list_testcase if x in i["request"]["url"]]
    if(api_of_extension["apiCall"] in list_name_api_of_behavior):

        if("webRequest" in api_of_extension["other"]):
            cancel_stt = json.loads(api_of_extension["other"]["webRequest"])
            if("cancel" in cancel_stt):
                matches = [x for x in list_args if x in api_of_extension["pageUrl"]]
                if(cancel_stt["cancel"] == True and len(matches) != 0 ):
                    return True
        #if(list_args in api_of_extension["apiCall"] ) :
        #    print(api_of_extension)
    return False

def DeleteReponseHeaderTracking(api_of_extension):
    # Kiem tra activityType co phai web_request hay khong
    # Neu phai thi chuyen sang kiem tra apiCall co phai la webRequest.onHeadersReceived
    # Kiem tra thuoc tinh other co chua webRequest["deleted_response_headers"]
    # Kiem tra webRequest["deleted_response_headers"] co chua cac gia tri header bao mat hay khong

    _behavior_info = GetBehaviorMalicious("deleted_response_headers")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []

    for api_behavior in list_api_behavior:
        # Kiem tra api trong list_name_api_of_behavior hay khong chua, khong add cac api null
        # Neu chua co thi them vao list
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])

    # Checking
    if((api_of_extension["apiCall"] in list_name_api_of_behavior)):
        if("webRequest" in api_of_extension["other"]):
            if("deleted_response_headers" in api_of_extension["other"]["webRequest"]):
                return True
    return False

def InjectsDynamicJsTracking(api_of_extension):
    _behavior_info = GetBehaviorMalicious("injects_dynamic_javascript")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    list_args = []
    for api_behavior in list_api_behavior:
        # Kiem tra api trong list_name_api_of_behavior hay khong chua, khong add cac api null
        # Neu chua co thi them vao list
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])
        list_args = (api_behavior["args"])

    #Tracking APi
    if((api_of_extension["apiCall"] in list_name_api_of_behavior)):
        for args_in_apicall in  json.loads(api_of_extension["args"]):
            if([x for x in list_args if x in args_in_apicall]):
                return True
    return False

def GetAllCookiesTracking(api_of_extension):
    _behavior_info = GetBehaviorMalicious("get_all_cookies")
    for api_of_behavior in (_behavior_info):
        if "behavior" in api_of_behavior:
            list_api_behavior = api_of_behavior["behavior"]

    list_name_api_of_behavior = []
    for api_behavior in list_api_behavior:
        # Kiem tra api trong list_name_api_of_behavior hay khong chua, khong add cac api null
        # Neu chua co thi them vao list
        if(api_behavior["apiCall"] != "" and api_behavior["apiCall"] not in list_name_api_of_behavior):
            list_name_api_of_behavior.append(api_behavior["apiCall"])

    #Tracking APi
    if((api_of_extension["apiCall"] in list_name_api_of_behavior)):
        return True
    return False

white_list_http = ["https://fbsbx.com/ajax/bz","https://www.paypal.com/signin/client-log","https://www.amazon.com/gp/recent-history-footer/external/rhf-handler.html","https://www.paypal.com/auth/verifychallenge"]
white_list_testcase =["facebook","fb","google","timo","paypal","amazon","shopee","twitter","bitdefender","norton","kaspersky","eset","myvisualiq"]
def NetworkRequest4xxTracking(idx):
    http_request_4xx = []
    mycol = init_database("NETWORK")
    my_network = mycol.find({"idx":idx})
    for info in my_network:
        path_file_network = info["Path"]
        with open(path_file_network, 'r') as f:
            entry = json.load(f)
        entries = entry["log"]["entries"]
        for i in entries:
            if(i["response"]["status"] >= 400 and i["response"]["status"] < 500):
                matches = [x for x in white_list_testcase if x in i["request"]["url"]]
                if(len(matches)!=0):
                    continue
                if(i["request"]["url"] not in white_list_http):
                    http_request_4xx.append({i["request"]["url"]:i["response"]["status"]})
    return http_request_4xx

def DnsResponseTracking(idx):
    dns_domain_whitelist = []
    with open(r"G:\New\Extensions\KhoaLuan\source\sandbox\white_list_dns.json", 'r') as f:
        entry = json.load(f)
    dns_domain_whitelist = entry["domain"]

    dns_no_response = []
    mycol = init_database("DNS")
    list_dns_of_idx = mycol.find({"idx":idx})
    for dns_record in list_dns_of_idx:
        if(dns_record["request"]["qname"][:-1] in dns_domain_whitelist):
            continue
        if("response" not in dns_record):
            dns_no_response.append(dns_record)
    return dns_no_response

def AnalyzerOnlyOneExtension(idx):
    total_call = 0
    count_api = {}
    api_called = []
    # Get api called of chrome extension from mongodb with id
    # Count total api called
    # Save element of info to report

    list_api_from_database = GetApiCalledByExtension(idx)
    for api_call in list_api_from_database:
        api_called.append(api_call)
        total_call += 1
        if(api_call["apiCall"] in count_api.keys()):
            count_api[api_call["apiCall"]] += 1
        else:
            count_api[api_call["apiCall"]] = 1
    beauty_report = {"id": idx, "total_api": total_call, "apis": count_api,"api_called":api_called}
    print("==========================================")
    list_api = GetApiCalledByExtension(idx)
    uninstall_other_extension=[]
    prevents_extension_uninstall=[]
    keylogging_functionality=[]
    steal_information_form=[]
    block_antivirus_site=[]
    deleted_response_headers=[]
    injects_dynamic_javascript=[]
    get_all_cookies=[]
    http_request_4xx = []
    dns_no_response = []
    for api in list_api:
        if (UninstallBehaviorTracking(api)):
            uninstall_other_extension.append(api)
            continue
        # detect PreventsUninstallTracking

        if(PreventsUninstallTracking(api)):
            prevents_extension_uninstall.append(api)
            continue
        if(KeyloggerTracking(api)):
            keylogging_functionality.append(api)
            continue
        if(StealInformationFormTracking(api)):
            all_info_behavior = []
            all_info_behavior.append(api)
            find_activity = mycol.find({"extensionId": api["extensionId"],"pageUrl":api["pageUrl"],"activityType":"content_script"})
            for api_content_script in find_activity:
                all_info_behavior.append(api_content_script)
            steal_information_form.append(all_info_behavior)
            continue
        if(BlockAntiVirusSiteTracking(api)):
            block_antivirus_site.append(api)
            continue
        if(DeleteReponseHeaderTracking(api)):
            deleted_response_headers.append(api)
            continue
        if(InjectsDynamicJsTracking(api)):
            injects_dynamic_javascript.append(api)
            continue
        if(GetAllCookiesTracking(api)):
            get_all_cookies.append(api)
            continue

    http_request_4xx = NetworkRequest4xxTracking(idx)
    dns_no_response = DnsResponseTracking(idx)
    beauty_report["uninstall_other_extension"] = uninstall_other_extension
    beauty_report["prevents_extension_uninstall"] = prevents_extension_uninstall
    beauty_report["keylogging_functionality"] = keylogging_functionality
    beauty_report["steal_information_form"] = steal_information_form
    beauty_report["block_antivirus_site"] = block_antivirus_site
    beauty_report["deleted_response_headers"] = deleted_response_headers
    beauty_report["injects_dynamic_javascript"] = injects_dynamic_javascript
    beauty_report["get_all_cookies"] = get_all_cookies
    beauty_report["http_request_4xx"] = http_request_4xx
    beauty_report["dns_no_response"] = dns_no_response
    col = init_database("REPORT")
    col.insert(beauty_report,check_keys=False)
    print("[+] Inserted ",idx)

def AnalyzerAllExtension():
# Doc tung report trong Database "REPORT" bang mycol.find
    malicious = 0
    suspicious = 0
    clean = 0
    top_10_extension_malicious = []
    top_10_api_called = {}
    info = {}
    uninstall_other_extension= 0
    prevents_extension_uninstall=0
    keylogging_functionality=0
    steal_information_form=0
    block_antivirus_site=0
    deleted_response_headers=0
    injects_dynamic_javascript=0
    get_all_cookies=0
    http_request_4xx = 0
    dns_no_response = 0
    mycol = init_database("REPORT")
    a = mycol.estimated_document_count()
    print("[+] Total %d reports"%(a))
    for ext in mycol.find():
        is_malicious = False
        is_suspicious = False
        count = 0
        behavior = []
        if(len(ext["uninstall_other_extension"]) != 0):
            count += 1
            uninstall_other_extension +=1
            behavior.append("uninstall_other_extension")
            is_malicious = True

        if(len(ext["prevents_extension_uninstall"]) != 0):
            count += 1
            prevents_extension_uninstall +=1
            behavior.append("prevents_extension_uninstall")
            is_malicious = True

        if(len(ext["keylogging_functionality"]) != 0):
            count += 1
            keylogging_functionality +=1
            behavior.append("keylogging_functionality")
            is_malicious = True

        if(len(ext["steal_information_form"]) != 0):
            count += 1
            steal_information_form +=1
            behavior.append("steal_information_form")
            is_malicious = True

        if(len(ext["block_antivirus_site"]) != 0):
            count += 1
            block_antivirus_site +=1
            behavior.append("block_antivirus_site")
            is_malicious = True

        if(len(ext["deleted_response_headers"]) != 0):
            count += 1
            deleted_response_headers +=1
            behavior.append("deleted_response_headers")
            is_malicious = True

        if(len(ext["injects_dynamic_javascript"]) != 0):
            count += 1
            injects_dynamic_javascript +=1
            behavior.append("injects_dynamic_javascript")
            if(is_malicious == False):
                is_suspicious = True

        if(len(ext["get_all_cookies"]) != 0):
            count += 1
            get_all_cookies +=1
            behavior.append("get_all_cookies")
            if(is_malicious == False):
                is_suspicious = True

        if(len(ext["http_request_4xx"]) != 0):
            count += 1
            http_request_4xx +=1
            behavior.append("http_request_4xx")
            if(is_malicious == False):
                is_suspicious = True

        if(len(ext["dns_no_response"]) != 0):
            #count += 1
            dns_no_response +=1
            #behavior.append("dns_no_response")


        if(is_malicious):
            malicious +=1
        elif(is_suspicious):
            suspicious +=1
        else:
            clean +=1
        info["id"] = ext["id"]
        info["count"] = count
        info["behavior"] = behavior
        datatest = info.copy()
        top_10_extension_malicious.append(datatest)

        for api_name in ext["apis"]:
            if(api_name not in top_10_api_called):
                top_10_api_called[api_name] =ext["apis"][api_name]
            else:
                top_10_api_called[api_name] += ext["apis"][api_name]

    top_10_api_called = Counter(top_10_api_called)
    top10 = dict(top_10_api_called.most_common(10))
    print("[+] Malicious:",malicious)
    print(" |- uninstall_other_extension:",uninstall_other_extension)
    print(" |- prevents_extension_uninstall:",prevents_extension_uninstall)
    print(" |- keylogging_functionality:",keylogging_functionality)
    print(" |- steal_information_form:",steal_information_form)
    print(" |- block_antivirus_site:",block_antivirus_site)
    print(" |- deleted_response_headers:",deleted_response_headers)
    print("[+] Suspicious:",suspicious)
    print(" |- injects_dynamic_javascript:",injects_dynamic_javascript)
    print(" |- get_all_cookies:",get_all_cookies)
    print(" |- http_request_4xx:",http_request_4xx)
    print("[+] Clean:",clean)
    print(" |- dns_no_response:",dns_no_response)
    print("[*] Top 10 API Called:")
    print(top10)
    print("[*] Top 10 Extension:")
    print(sorted(top_10_extension_malicious, key = lambda i: i['count'],reverse=True)[:10])

if __name__ == "__main__":
    AnalyzerAllExtension()