import os
import csv
import pandas as pd
import pytz
from datetime import datetime, timedelta
import requests

'''
checking missing data
'''

# Define the absolute path to the root directory where the files are located
root_directory_path = "D:/WS/VsCode/AD_DL_CSV/01_DivideData/src_filtered/03_folder_sorting_by_name"

def extract_month_from_filename(filename):
    parts = filename.split('_')
    return int(parts[-1].split('.')[0])

def construct_expected_timestamps(month):
    year = 2017
    start_date = datetime(year, month, 1, 0, 0, 0, tzinfo=pytz.FixedOffset(480))

    if month == 12:
        end_date = datetime(year+1, 1, 1, 0, 0, 0,tzinfo=pytz.FixedOffset(480))
    else:
        next_month = month + 1
        end_date = datetime(year, next_month, 1, 0, 0, 0,tzinfo=pytz.FixedOffset(480))

    return pd.date_range(start=start_date, end=end_date, freq='5T')

def check_timestamp_intervals_and_station_id(csv_file):
    month = extract_month_from_filename(csv_file)
    expected_timestamps = construct_expected_timestamps(month)

    df = pd.read_csv(csv_file)
    df['Timestamp'] = pd.to_datetime(df['Timestamp'])

    # Check for missing timestamps
    missing_timestamps = expected_timestamps[~expected_timestamps.isin(df['Timestamp'])]

    return missing_timestamps

'''
Fetch Data
'''
def fetch_data(date_time, query_type):
    # Query locations
    #   air-temperature
    #   relative-humidity
    #   rainfall
    #   wind-speed
    url = f"https://api.data.gov.sg/v1/environment/{query_type}?date_time={date_time}"

    response = requests.get(url)
    if response.status_code == 200:
        return response.json()
    elif 500 <= response.status_code < 600:
        print(f"{response.status_code}: The server did not respond in time. Retrying...")
        return fetch_data(date_time, query_type)
    else:
        print(f"Failed to fetch data. Status code: {response.status_code}")
        return None

def export_to_csv(data, filename):
    if data is None:
        return

    with open(filename, 'a', newline='') as csvfile:
        writer = csv.writer(csvfile)
        for item in data['items']:
            timestamp = item['timestamp']
            readings = item['readings']
            for reading in readings:
                station_id = reading.get('station_id', 'UNKNOWN')
                myValue = reading.get('value', 'NULL')
                writer.writerow([timestamp, station_id, myValue])


def initiate_download(filename, label, formatted_time, query_type):
    #TODO: change label

    with open(filename, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(["Timestamp", "Station ID", label])

        date_time_to_fetch = formatted_time.strftime("%Y-%m-%dT%H:%M:%S")
        json_data = fetch_data(date_time_to_fetch, query_type)
        if json_data:
            export_to_csv(json_data, filename)
            print(f"Data for {filename}, {date_time_to_fetch} exported successfully.")



# Loop through each subfolder and its files to process the data
for subfolder in ["windspeed", "airtemperature", "humidity", "rainfall"]:
    subfolder_path = os.path.join(root_directory_path, subfolder)
    files = os.listdir(subfolder_path)

    for file in files:
        if file.endswith(".csv"):
            file_path = os.path.join(subfolder_path, file)  # need to append the entire thing
            missing = check_timestamp_intervals_and_station_id(file_path)



            # Split the file_name at underscores
            parts = file.split('_')
            # Combine the parts before the second underscore and append ".csv"
            new_file_name = '_'.join(parts[:2]) + ".csv"
            new_root_path = "D:/WS/VsCode/AD_DL_CSV/01_DivideData/src_filtered/04_dl_missing"
            new_file_path = os.path.join( new_root_path, new_file_name)

            if subfolder == "windspeed":
                label = "Wind Speed (knots)"
                query_type = "wind-speed"

            elif subfolder == "rainfall":
                label = "Rainfall (mm)"
                query_type = "rainfall"

            elif subfolder == "airtemperature":
                label = "Humidity (pct)"
                query_type = "relative-humidity"

            else:
                label = "Air Temperature (deg C)"
                query_type = "air-temperature"

            for timestamp in missing:
                initiate_download(new_file_path, label, timestamp, query_type)