# -*- coding: utf-8 -*-
"""
Created on Mon Feb  3 12:54:11 2020

@original author: Saniya Khan
@adopted code: Valentin Heimhuber

This code uses a coords.csv file with a list of lat lon locations to extract SILO gridded climatology data at each location and
writes it into a csv file

More info about how to do the datadrills via api are provided here: https://www.longpaddock.qld.gov.au/silo/api-documentation/

SILO gridded data are 0.05° × 0.05° or 50x50km in resolution

More information about the interpolated & gridded SILO datasets can be found here: https://www.longpaddock.qld.gov.au/silo/faq/#faq5
"""
from __future__ import unicode_literals
import urllib.request
import urllib.parse
import pandas as pd
from io import StringIO
from itertools import repeat
api_url = 'https://www.longpaddock.qld.gov.au/cgi-bin/silo'

path_to_coordscsv_file = 'J:/Project/wrl2018064 Fisheries RAP/04_Working/05_Modelling/RMA/HEMIP/Global_Data/Climatology/SILO/'
#replace by actual geocodes
#geocode=pd.DataFrame([['YANKALILLA','-35.494262', '138.362596'], 
#              ['PORT WAKEFIELD','-34.185349', '138.155379']],columns=['Brigade','latitude','longitude'])

list_of_weather=[] 
weather=pd.DataFrame()
def getGeocode():
    geocode= pd.read_csv(path_to_coordscsv_file + "coords.csv")
    print(geocode.columns)
    geocode.set_index('Brigade')
    #print("dfdfdfD")
    #print(geocode)
    return geocode;
def buildUrl(lat,long):
    params = {
    'format': 'alldata',
    'lat': str(lat),
    'lon': str(long),
    'start': '20090101',
    'finish': '20181231',
    'username': 'sk3862@drexel.edu',
    'password': 'silo'
    }
    url = api_url + '/DataDrillDataset.php?' + urllib.parse.urlencode(params)
    return url    
def sendRequest(url):
    with urllib.request.urlopen(url) as remote:
        data = remote.read()
        s=str(data,'utf-8')
        data_formatted = StringIO(s) 
        df=pd.read_csv(data_formatted)
    return df
def getData(lat,long) :

    return weather

#lat long index
geocode=getGeocode()
for i in range(len(geocode)):
    print(i)
    brigade=[geocode.loc[i,'Brigade']]
    print(brigade)
    url=buildUrl(np.round(geocode.loc[i,'latitude'],4),  np.round(geocode.loc[i,'longitude'], 4)) #url for lat long
    df=sendRequest(url) #ping the australian websiten
    if  i==0: 
        headr=df.iloc[46,:]
        headr.reset_index(inplace=True, drop=True)
        headr.replace('\s+', ',',regex=True,inplace=True) #separate out the header
        headr[0]=headr[0]+",Brigade"+",Latitude"+",Longitude";
        list_of_weather.append(headr) 
    df=df[47:(len(df)-1)]#cleaning remove header, indexes
    df.replace('\s+', ',',regex=True,inplace=True) #make csv space delimited to comma delimited
    df=df.iloc[:,-1]#cleaning 
    df.name=brigade[0]
    formatted_data=df+","+brigade[0]+","+str(geocode.loc[i,'latitude'])+","+str(geocode.loc[i,'longitude'])
    type(formatted_data)
    list_of_weather.append(formatted_data)#combine for different locations
    #weather=pd.concat(list_of_weather)
#weather = [getData(x, y) for x, y in zip(geocode['latittude'], geocode['longitude'])]

    #reformat into a nice dataframe with colnames and save that as CSV
    #col_names=["Date (yyyymmdd)","Day","T.Max(oC)","Smx","T.Min(oC)","Smn","Rain (mm)","Srn","Evap(mm)","Sev", "Radn(MJ/m2)","Ssl" ,"VP (hPA)","Svp","RHmaxT(%)" ,"RHminT(%)" ,"Date2(ddmmyyyy)","Brigade","Latitude","Longitude"]
    col_names=["Date (yyyymmdd)","Day", "Date2(ddmmyyyy)",    "T.Max(oC)","Smx","T.Min(oC)","Smn","Rain (mm)","Srn","Evap(mm)","Sev", "Radn(MJ/m2)","Ssl" ,"VP (hPA)","Svp","RHmaxT(%)" ,"RHminT(%)" ,
               'FAO56(mm)', 'Mlake(mm)','(Mpot(mm)'   ,'Mact(mm)'   ,'Mwet(mm)'  ,'Span(mm)'   ,'Ssp()'   ,'EvSp(mm)' ,'Ses()' ,'MSLPres(hPa)' ,'Sp()', "Brigade","Latitude","Longitude"]      
    df = pd.DataFrame(formatted_data.str.split(',',n=30).tolist(),
                           columns = col_names)
    df = df.iloc[1:]
    print(df.head())
    df.to_csv(path_to_coordscsv_file + '_SILO_weather_alldata_' + brigade[0] + '.csv')