import requests
import json
import time
import csv
import pandas

start=250

with open('C:/Users/apskaita3/Desktop/number2.txt', "r") as f:

    start= f.readlines()

start=int(start[0])

start=start + 70


results = {"item": {}}

# Todo load json
for i in range(0,9800): #<----- Just change range here to increase number of requests
    URL = f"https://api.news.eu.nasdaq.com/news/query.action?type=handleResponse&showAttachments=true&showCnsSpecific=true&showCompany=true&countResults=false&freeText=&company=&market=Main%20Market%2C+Helsinki&cnscategory=&fromDate=&toDate=&globalGroup=exchangeNotice&globalName=NordicMainMarkets&displayLanguage=en&language=en&timeZone=CET&dateMask=yyyy-MM-dd+HH%3Amm%3Ass&limit=50000000&start={i}&dir=ASC"
    r = requests.get(url = URL)
    #time.sleep(1)
    res = r.text.replace("handleResponse(", "")
    #print(res)
    #print(f'r is {r}')
    res_json = json.loads(res)
    #print(res_json)
    data = res_json
    a=i+1
    #print(data)
    print("Doing: " + str(i + 1) + "th")
    #data = r.json()

    downloaded_entries = data["results"]["item"]
    new_entries = [d for d in downloaded_entries if d["headline"] not in results["item"]]
    start=str(start)

    for entry in new_entries:
        if entry["market"] == 'Main Market, Helsinki' and entry["published"]>="2021-10-20 06:30:00":
            headline = entry["headline"].strip()
            published = entry["published"]
            market=entry["market"]
            market="Main Market, Helsinki"
            results["item"][headline] = {"company": entry["company"], "messageUrl": entry["messageUrl"], "published": entry["published"], "headline": headline}
            print(entry['market'])
            #time.sleep(5)
            print(f"Market: {market}/nDate: {published}/n")
            #print( results["item"][headline] )
            #print(results)
            #print(json.dumps({"item": list(results["item"].values())}, indent = 4))

with open("C:/Users/apskaita3/Finansų analizės ir valdymo sprendimai, UAB/Rokas Toomsalu - Power BI analitika/Integracijos/1_Public comapnies analytics/Databasesets/Others/market_news_helsinki.json", "w") as outfile:
    json_object = json.dumps({"item": list(results["item"].values())}, indent = 4)
    outfile.write(json_object)
    #print(json_object)
with open("C:/Users/apskaita3/Desktop/number2.txt", "w") as outfile1:
    outfile1.write(start)  # type: ignore

"""""
with open('C:/Users/apskaita3/Finansų analizės ir valdymo sprendimai, UAB/Rokas Toomsalu - Power BI analitika/Integracijos/1_Public comapnies analytics/Databasesets/Others/market_news_helsinki.json') as json_file:
    data = json.load(json_file)


employee_data = data['item']



data_file = open("C:/Users/apskaita3/Desktop/data_file9.csv", 'a',newline='', encoding="utf-8")


csv_writer = csv.writer(data_file)


count=0

for emp in employee_data:
    if count == 0:

        # Writing headers of CSV file
        header = emp.keys()
        csv_writer.writerow(header)
        count += 1

    print(emp.values())
    # Writing data of CSV file
    csv_writer.writerow(emp.values())

data_file.close()


csvFile = pandas.read_csv('C:/Users/apskaita3/Desktop/data_file9.csv', sep=',',encoding="utf-8")

csvFile=csvFile.drop_duplicates()
#print(csvFile.iloc[0])
csvFile.to_csv("C:/Users/apskaita3/Desktop/data_file15.csv",index = False)



with open("C:/Users/apskaita3/Desktop/data_file15.csv","r",encoding="utf-8") as inputfile, open("C:/Users/apskaita3/Desktop/data_file16.csv","a",encoding="utf-8") as outputfile:
    lines = inputfile.readlines()
    outputfile.write(lines[0])
    for i in range(1, len(lines)):
        if lines[i] != lines[0]:
            outputfile.write(lines[i])


csvFile1 = pandas.read_csv('C:/Users/apskaita3/Desktop/data_file9.csv', sep=',',encoding="utf-8")
csvFile1=csvFile.drop_duplicates()
csvFile1 = csvFile1[csvFile1.company != "company"]
csvFile1.to_csv("C:/Users/apskaita3/Desktop/data_file16.csv",index = False)






# Function to convert a CSV to JSON
# Takes the file paths as arguments
def make_json(csvFilePath, jsonFilePath):

    # create a dictionary
    data = {}

    # Open a csv reader called DictReader
    with open(csvFilePath, encoding='utf-8') as csvf:
        csvReader = csv.DictReader(csvf)

        # Convert each row into a dictionary
        # and add it to data
        for rows in csvReader:

            # Assuming a column named 'No' to
            # be the primary key
            key = rows['headline']
            data[key] = rows

    # Open a json writer, and use the json.dumps()
    # function to dump data
    with open(jsonFilePath, 'w', encoding='utf-8') as jsonf:
        jsonf.write(json.dumps(data, indent=4))

# Driver Code

# Decide the two file paths according to your
# computer system
csvFilePath = r'C:/Users/apskaita3/Desktop/data_file16.csv'
jsonFilePath = r'C:/Users/apskaita3/Finansų analizės ir valdymo sprendimai, UAB/Rokas Toomsalu - Power BI analitika/Integracijos/1_Public comapnies analytics/Databasesets/Others/market_news_helsinki.json'

# Call the make_json function
make_json(csvFilePath, jsonFilePath)


















#input("Press")
#print(json_object)
"""