|
|
|
|
|
# -*- coding: utf-8 -*-
|
|
|
|
|
|
"""
|
|
|
|
|
|
Created on Mon Feb 3 12:54:11 2020
|
|
|
|
|
|
|
|
|
|
|
|
@original author: Saniya Khan
|
|
|
|
|
|
@adopted code: Valentin Heimhuber
|
|
|
|
|
|
|
|
|
|
|
|
This code uses a coords.csv file with a list of lat lon locations to extract SILO gridded climatology data at each location and
|
|
|
|
|
|
writes it into a csv file
|
|
|
|
|
|
|
|
|
|
|
|
More info about how to do the datadrills via api are provided here: https://www.longpaddock.qld.gov.au/silo/api-documentation/
|
|
|
|
|
|
|
|
|
|
|
|
SILO gridded data are 0.05° × 0.05° or 50x50km in resolution
|
|
|
|
|
|
|
|
|
|
|
|
More information about the interpolated & gridded SILO datasets can be found here: https://www.longpaddock.qld.gov.au/silo/faq/#faq5
|
|
|
|
|
|
"""
|
|
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
import urllib.request
|
|
|
|
|
|
import urllib.parse
|
|
|
|
|
|
import pandas as pd
|
|
|
|
|
|
from io import StringIO
|
|
|
|
|
|
from itertools import repeat
|
|
|
|
|
|
api_url = 'https://www.longpaddock.qld.gov.au/cgi-bin/silo'
|
|
|
|
|
|
|
|
|
|
|
|
path_to_coordscsv_file = 'J:/Project/wrl2018064 Fisheries RAP/04_Working/05_Modelling/RMA/HEMIP/Global_Data/Climatology/SILO/'
|
|
|
|
|
|
#replace by actual geocodes
|
|
|
|
|
|
#geocode=pd.DataFrame([['YANKALILLA','-35.494262', '138.362596'],
|
|
|
|
|
|
# ['PORT WAKEFIELD','-34.185349', '138.155379']],columns=['Brigade','latitude','longitude'])
|
|
|
|
|
|
|
|
|
|
|
|
list_of_weather=[]
|
|
|
|
|
|
weather=pd.DataFrame()
|
|
|
|
|
|
def getGeocode():
|
|
|
|
|
|
geocode= pd.read_csv(path_to_coordscsv_file + "coords.csv")
|
|
|
|
|
|
print(geocode.columns)
|
|
|
|
|
|
geocode.set_index('Brigade')
|
|
|
|
|
|
#print("dfdfdfD")
|
|
|
|
|
|
#print(geocode)
|
|
|
|
|
|
return geocode;
|
|
|
|
|
|
def buildUrl(lat,long):
|
|
|
|
|
|
params = {
|
|
|
|
|
|
'format': 'alldata',
|
|
|
|
|
|
'lat': str(lat),
|
|
|
|
|
|
'lon': str(long),
|
|
|
|
|
|
'start': '20090101',
|
|
|
|
|
|
'finish': '20181231',
|
|
|
|
|
|
'username': 'sk3862@drexel.edu',
|
|
|
|
|
|
'password': 'silo'
|
|
|
|
|
|
}
|
|
|
|
|
|
url = api_url + '/DataDrillDataset.php?' + urllib.parse.urlencode(params)
|
|
|
|
|
|
return url
|
|
|
|
|
|
def sendRequest(url):
|
|
|
|
|
|
with urllib.request.urlopen(url) as remote:
|
|
|
|
|
|
data = remote.read()
|
|
|
|
|
|
s=str(data,'utf-8')
|
|
|
|
|
|
data_formatted = StringIO(s)
|
|
|
|
|
|
df=pd.read_csv(data_formatted)
|
|
|
|
|
|
return df
|
|
|
|
|
|
def getData(lat,long) :
|
|
|
|
|
|
|
|
|
|
|
|
return weather
|
|
|
|
|
|
|
|
|
|
|
|
#lat long index
|
|
|
|
|
|
geocode=getGeocode()
|
|
|
|
|
|
for i in range(len(geocode)):
|
|
|
|
|
|
print(i)
|
|
|
|
|
|
brigade=[geocode.loc[i,'Brigade']]
|
|
|
|
|
|
print(brigade)
|
|
|
|
|
|
url=buildUrl(np.round(geocode.loc[i,'latitude'],4), np.round(geocode.loc[i,'longitude'], 4)) #url for lat long
|
|
|
|
|
|
df=sendRequest(url) #ping the australian websiten
|
|
|
|
|
|
if i==0:
|
|
|
|
|
|
headr=df.iloc[46,:]
|
|
|
|
|
|
headr.reset_index(inplace=True, drop=True)
|
|
|
|
|
|
headr.replace('\s+', ',',regex=True,inplace=True) #separate out the header
|
|
|
|
|
|
headr[0]=headr[0]+",Brigade"+",Latitude"+",Longitude";
|
|
|
|
|
|
list_of_weather.append(headr)
|
|
|
|
|
|
df=df[47:(len(df)-1)]#cleaning remove header, indexes
|
|
|
|
|
|
df.replace('\s+', ',',regex=True,inplace=True) #make csv space delimited to comma delimited
|
|
|
|
|
|
df=df.iloc[:,-1]#cleaning
|
|
|
|
|
|
df.name=brigade[0]
|
|
|
|
|
|
formatted_data=df+","+brigade[0]+","+str(geocode.loc[i,'latitude'])+","+str(geocode.loc[i,'longitude'])
|
|
|
|
|
|
type(formatted_data)
|
|
|
|
|
|
list_of_weather.append(formatted_data)#combine for different locations
|
|
|
|
|
|
#weather=pd.concat(list_of_weather)
|
|
|
|
|
|
#weather = [getData(x, y) for x, y in zip(geocode['latittude'], geocode['longitude'])]
|
|
|
|
|
|
|
|
|
|
|
|
#reformat into a nice dataframe with colnames and save that as CSV
|
|
|
|
|
|
#col_names=["Date (yyyymmdd)","Day","T.Max(oC)","Smx","T.Min(oC)","Smn","Rain (mm)","Srn","Evap(mm)","Sev", "Radn(MJ/m2)","Ssl" ,"VP (hPA)","Svp","RHmaxT(%)" ,"RHminT(%)" ,"Date2(ddmmyyyy)","Brigade","Latitude","Longitude"]
|
|
|
|
|
|
col_names=["Date (yyyymmdd)","Day", "Date2(ddmmyyyy)", "T.Max(oC)","Smx","T.Min(oC)","Smn","Rain (mm)","Srn","Evap(mm)","Sev", "Radn(MJ/m2)","Ssl" ,"VP (hPA)","Svp","RHmaxT(%)" ,"RHminT(%)" ,
|
|
|
|
|
|
'FAO56(mm)', 'Mlake(mm)','(Mpot(mm)' ,'Mact(mm)' ,'Mwet(mm)' ,'Span(mm)' ,'Ssp()' ,'EvSp(mm)' ,'Ses()' ,'MSLPres(hPa)' ,'Sp()', "Brigade","Latitude","Longitude"]
|
|
|
|
|
|
df = pd.DataFrame(formatted_data.str.split(',',n=30).tolist(),
|
|
|
|
|
|
columns = col_names)
|
|
|
|
|
|
df = df.iloc[1:]
|
|
|
|
|
|
print(df.head())
|
|
|
|
|
|
df.to_csv(path_to_coordscsv_file + '_SILO_weather_alldata_' + brigade[0] + '.csv')
|
