#fully working set of codes for generating the NARCLIM

Delta CSVs to then generate the climate deviation plots
by combining with observational time seris
Development1
tinoheimhuber 7 years ago
parent b2fc7135d9
commit 8a771b65b6

@ -2,8 +2,9 @@
#####################################---------------------------------- #####################################----------------------------------
#Last Updated - March 2018 #Last Updated - March 2018
#@author: z5025317 Valentin Heimhuber #@author: z5025317 Valentin Heimhuber
#code for creating climate prioritization plots for NARCLIM variables. #code for creating future climate variability deviation plots for NARCLIM variables.
#Inputs: Uses CSV files that contain all 12 NARCLIM model runs time series for 1 grid cell created with: P1_NARCliM_NC_to_CSV_CCRC_SS.py #Inputs: Uses CSV files that containthe deltas of all 12 NARCLIM models for 1 grid cell at the site of interest, generated with P1_NARCLIM_plots_Windows.py
#This code is used only for the NARCLIM variables - a separate code is used for ocean variables etc that are not part of the NARCLIM ensemble
#####################################---------------------------------- #####################################----------------------------------
#Load packages #Load packages
#####################################---------------------------------- #####################################----------------------------------
@ -27,10 +28,10 @@ os.chdir('C:/Users/z5025317/WRL_Postdoc/Projects/Paper#1/')
#set input parameters #set input parameters
Base_period_start = '1986-01-01' Base_period_start = '1986-01-01'
Base_period_end = '2005-01-01' #use last day that's not included in period as < is used for subsetting Base_period_end = '2005-01-01' #use last day that's not included in period as < is used for subsetting
Estuary = 'Tweed' # 'Belongil' Estuary = 'Terrigal' # 'Belongil'
Clim_var_type = "*" # '*' will create pdf for all variables in folder Clim_var_type = "*" # '*' will create pdf for all variables in folder
Clim_var_type = "tasmax*" # '*' will create pdf for all variables in folder Clim_var_type = "wssmean*" # '*' will create pdf for all variables in folder
Present_Day_Clim_Var = 'Rainfall' Present_Day_Clim_Var = 'Wind' #MaxT, MinT, Rainfall, ET
#####################################---------------------------------- #####################################----------------------------------
#set directory path for output files #set directory path for output files
@ -44,30 +45,58 @@ if not os.path.exists(output_directory):
print('-------------------') print('-------------------')
Clim_Var_CSVs = glob.glob('./Output/' + Estuary + '/' + Estuary + '_' + Clim_var_type + '*') Clim_Var_CSVs = glob.glob('./Output/' + Estuary + '/' + Estuary + '_' + Clim_var_type + '*')
#read CSV file #read CSV file
for clim_var_csv_path in Clim_Var_CSVs: clim_var_csv_path = Clim_Var_CSVs[0]
Filename = os.path.basename(os.path.normpath(clim_var_csv_path)) Filename = os.path.basename(os.path.normpath(clim_var_csv_path))
Clim_var_type = Filename.split('_', 2)[1] Clim_var_type = Filename.split('_', 2)[1]
print(Clim_var_type) print(Clim_var_type)
Ensemble_Delta_full_df = pd.read_csv(clim_var_csv_path, index_col=0, parse_dates = True) Ensemble_Delta_full_df = pd.read_csv(clim_var_csv_path, index_col=0, parse_dates = True)
#Ensemble_Delta_full_df = pd.to_numeric(Ensemble_Delta_full_df) #Ensemble_Delta_full_df = pd.to_numeric(Ensemble_Delta_full_df)
#
#load present day climate data for the same variable #load present day climate data for the same variable
if Clim_var_type == 'evspsblmean': #ET time series that we have is not in the same format as the other variables, hence the different treatment
Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_' + Present_Day_Clim_Var + '*csv') Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_' + Present_Day_Clim_Var + '*csv')
Present_day_df = pd.read_csv(Present_day_Var_CSV[0]) Present_day_df = pd.read_csv(Present_day_Var_CSV[0])
if Clim_var_type == 'evspsblmean':
Dates = pd.to_datetime(Present_day_df.Date) Dates = pd.to_datetime(Present_day_df.Date)
Present_day_df.index = Dates Present_day_df.index = Dates
Present_day_df = Present_day_df.iloc[:,1] Present_day_df = Present_day_df.iloc[:,1]
Present_day_df = Present_day_df.replace(r'\s+', np.nan, regex=True) Present_day_df = Present_day_df.replace(r'\s+', np.nan, regex=True)
Present_day_df = pd.to_numeric(Present_day_df) Present_day_df = pd.to_numeric(Present_day_df)
Present_day_df.index = Dates
else: [minplotDelta, maxplotDelta]=[50,50]
#for tasmean, observed min and max T need to be converted into mean T
elif Clim_var_type == 'tasmean':
Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_MaxT*csv')
Present_day_df = pd.read_csv(Present_day_Var_CSV[0])
Dates = pd.to_datetime(Present_day_df.Year*10000+Present_day_df.Month*100+Present_day_df.Day,format='%Y%m%d')
Present_day_df.index = Dates
Present_day_MaxT_df = Present_day_df.iloc[:,5]
Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_MinT*csv')
Present_day_df = pd.read_csv(Present_day_Var_CSV[0])
Dates = pd.to_datetime(Present_day_df.Year*10000+Present_day_df.Month*100+Present_day_df.Day,format='%Y%m%d')
Present_day_df.index = Dates
Present_day_MinT_df = Present_day_df.iloc[:,5]
Present_day_df = (Present_day_MaxT_df + Present_day_MinT_df)/2
[minplotDelta, maxplotDelta]=[1,2]
elif Clim_var_type == 'tasmax':
[minplotDelta, maxplotDelta]=[1,2]
elif Clim_var_type == 'wssmean' or Clim_var_type == 'wss1Hmaxtstep':
Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_' + Present_Day_Clim_Var + '*csv')
Present_day_df = pd.read_csv(Present_day_Var_CSV[0])
Present_day_df.index = Present_day_df[['Year', 'Month', 'Day', 'Hour']].apply(lambda s : datetime(*s),axis = 1)
Present_day_df = Present_day_df.filter(regex= 'm/s')
Present_day_df = Present_day_df.replace(r'\s+', np.nan, regex=True)
Present_day_df['Wind speed measured in m/s'] = Present_day_df['Wind speed measured in m/s'].convert_objects(convert_numeric=True)
[minplotDelta, maxplotDelta]=[1, 1.5]
else:
Present_day_Var_CSV = glob.glob('./Data/Wheather_Station_Data/**/' + Estuary + '_' + Present_Day_Clim_Var + '*csv')
Present_day_df = pd.read_csv(Present_day_Var_CSV[0])
Dates = pd.to_datetime(Present_day_df.Year*10000+Present_day_df.Month*100+Present_day_df.Day,format='%Y%m%d') Dates = pd.to_datetime(Present_day_df.Year*10000+Present_day_df.Month*100+Present_day_df.Day,format='%Y%m%d')
Present_day_df.index = Dates Present_day_df.index = Dates
Present_day_df = Present_day_df.iloc[:,5] Present_day_df = Present_day_df.iloc[:,5]
#create seasonal sums etc. [minplotDelta, maxplotDelta]=[50,50]
if (Clim_var_type == 'pracc' or Clim_var_type == 'evspsblmean' or Clim_var_type == 'potevpmean' #create seasonal sums etc.
or Clim_var_type == 'pr1Hmaxtstep' or Clim_var_type == 'wss1Hmaxtstep'): if (Clim_var_type == 'pracc' or Clim_var_type == 'evspsblmean' or Clim_var_type == 'potevpmean'
or Clim_var_type == 'pr1Hmaxtstep' or Clim_var_type == 'wss1Hmaxtstep'):
Present_day_df_annual = Present_day_df.resample('A').sum() Present_day_df_annual = Present_day_df.resample('A').sum()
Present_day_df_annual = Present_day_df_annual.replace(0, np.nan) Present_day_df_annual = Present_day_df_annual.replace(0, np.nan)
Present_day_df_monthly = Present_day_df.resample('M').sum() Present_day_df_monthly = Present_day_df.resample('M').sum()
@ -76,18 +105,18 @@ for clim_var_csv_path in Clim_Var_CSVs:
Present_day_df_weekly = Present_day_df_weekly.replace(0, np.nan) Present_day_df_weekly = Present_day_df_weekly.replace(0, np.nan)
Fdf_Seas_means = Present_day_df.resample('Q-NOV').sum() #seasonal means Fdf_Seas_means = Present_day_df.resample('Q-NOV').sum() #seasonal means
Fdf_Seas_means = Fdf_Seas_means.replace(0, np.nan) Fdf_Seas_means = Fdf_Seas_means.replace(0, np.nan)
else: else:
Present_day_df_annual = Present_day_df.resample('A').mean() Present_day_df_annual = Present_day_df.resample('A').mean()
Present_day_df_monthly = Present_day_df.resample('M').mean() Present_day_df_monthly = Present_day_df.resample('M').mean()
Present_day_df_weekly = Present_day_df.resample('W').mean() Present_day_df_weekly = Present_day_df.resample('W').mean()
Fdf_Seas_means = Present_day_df.resample('Q-NOV').mean() #seasonal means Fdf_Seas_means = Present_day_df.resample('Q-NOV').mean() #seasonal means
#Loop through annual and seasons and create a deviation plot for each. #Loop through annual and seasons and create a deviation plot for each.
times = ['annual', 'DJF', 'MAM', 'JJA','SON'] times = ['annual', 'DJF', 'MAM', 'JJA','SON']
fig = plt.figure(figsize=(14,8)) fig = plt.figure(figsize=(14,8))
delta_all_df = pd.DataFrame() delta_all_df = pd.DataFrame()
i=1 i=1
for temp in times: for temp in times:
#subset the ensemble dataframe for the period used: #subset the ensemble dataframe for the period used:
if temp == 'annual': if temp == 'annual':
Ensemble_Delta_df = Ensemble_Delta_full_df.iloc[:,range(0,2)] Ensemble_Delta_df = Ensemble_Delta_full_df.iloc[:,range(0,2)]
@ -137,11 +166,11 @@ for clim_var_csv_path in Clim_Var_CSVs:
Plot_in_df_tp = Plot_in_df2.transpose() Plot_in_df_tp = Plot_in_df2.transpose()
#do the individual plots #do the individual plots
if temp == 'annual': if temp == 'annual':
xmin = int(min(Plot_in_df.min(axis=1))-1) xmin = int(min(Plot_in_df.min(axis=1))-minplotDelta)
xmax = int(max(Plot_in_df.max(axis=1))+2) xmax = int(max(Plot_in_df.max(axis=1))+maxplotDelta)
else: else:
xmin = int(min(Plot_in_df.min(axis=1))-1) xmin = int(min(Plot_in_df.min(axis=1))-minplotDelta)
xmax = int(max(Plot_in_df.max(axis=1))+2) xmax = int(max(Plot_in_df.max(axis=1))+maxplotDelta)
#define colour scheme #define colour scheme
#likert_colors = ['none', 'firebrick','firebrick','lightcoral','lightcoral'] #likert_colors = ['none', 'firebrick','firebrick','lightcoral','lightcoral']
likert_colors = ['none', 'darkblue', 'darkblue','cornflowerblue','cornflowerblue'] likert_colors = ['none', 'darkblue', 'darkblue','cornflowerblue','cornflowerblue']
@ -156,6 +185,8 @@ for clim_var_csv_path in Clim_Var_CSVs:
z.set_zorder(-1) z.set_zorder(-1)
z = plt.axhline(float(Present_Day_Mean+Present_Day_SD), linestyle='--', color='black', alpha=.5) z = plt.axhline(float(Present_Day_Mean+Present_Day_SD), linestyle='--', color='black', alpha=.5)
z.set_zorder(-1) z.set_zorder(-1)
z = plt.axhline(float(Present_Day_Mean), linestyle='--', color='red', alpha=.5)
z.set_zorder(-1)
plt.ylim(xmin, xmax) plt.ylim(xmin, xmax)
plt.title(Clim_var_type + ' ' + temp ) plt.title(Clim_var_type + ' ' + temp )
ax.grid(False) ax.grid(False)
@ -168,10 +199,10 @@ for clim_var_csv_path in Clim_Var_CSVs:
else: else:
i=i+1 i=i+1
print(i) print(i)
plt.show() plt.show()
out_file_name = Estuary + '_' + Clim_var_type + '_CC_prio_plot.png' out_file_name = Estuary + '_' + Clim_var_type + '_CC_prio_plot.png'
out_path = output_directory + '/' + out_file_name out_path = output_directory + '/' + out_file_name
fig.savefig(out_path) fig.savefig(out_path)

@ -1,13 +1,16 @@
Variables: Variables available from NARCLIM (output):
'evspsblmean' water_evaporation flux (actual ET) 'evspsblmean' water_evaporation flux (actual ET) long_name: Surface evaporation standard_name: water_evaporation_flux units: kg m-2 s-1
'potevpmean' potential ET 'potevpmean' potential ET water_potential_evaporation_flux kg m-2 s-1
'tasmean mean near surface temperature 'tasmean mean near surface temperature
tasmax maximum near surface temperature tasmax maximum near surface temperature
pracc precipitation daily precipitation sum (sum of convective prcacc and stratiform prncacc precip) pracc precipitation daily precipitation sum (sum of convective prcacc and stratiform prncacc precip)
pr1Hmaxtstep maximum 1 hour interval rainfall in a one day period pr1Hmaxtstep maximum 1 hour interval rainfall in a one day period
wssmean mean daily windspeed 'pr1Hmaxtstep' Max. 1-hour time-window moving averaged precipitation rate units: kg m-2 s-1 maximum 1-hour time-window moving averaged values from point values 60.0 second
'wss1Hmaxtstep' maximum 1 hour interval mean windspeeds in a one day period 'wss1Hmaxtstep' Max. 1-hour time-window moving averaged surface wind speed units: m s-1 maximum 1-hour time-window moving averaged values from point values 60.0 second
'wssmax' Surface wind speed standard_name: air_velocity units: m s-1 height: 10 m
'wssmean' Surface wind speed standard_name: air_velocity units: m s-1
Sites: Sites:
Northern NSW: Northern NSW:
@ -25,15 +28,47 @@ Towamba River: -37.1, 149.91
Nadgee Lake: -37.47, 149.97 Nadgee Lake: -37.47, 149.97
Bash-Code for netcdf interrogation: Bash-Code for netcdf interrogation:
latitude=-35.76 1st step: log into storm servers: Putty: hurricane.ccrc.unsw.edu.au or typhoon.ccrc.unsw.edu.au or cyclone.ccrc.unsw.edu.au + UNSW credentials (zID)
longitude = 150.25 In BASH:
module load python
latitude=-28.17
longitude=153.56
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'tasmean' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'tasmean' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'tasmax' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'tasmax' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'pr1Hmaxtstep' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'pr1Hmaxtstep' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'wssmean' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'wssmean' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'pracc' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'pracc' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'wss1Hmaxtstep' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'wss1Hmaxtstep' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat -32.843 --lon 151.706 --varName 'evspsblmean' --domain 'd02' --timestep 'DAY'; python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'evspsblmean' --domain 'd02' --timestep 'DAY';
python /srv/ccrc/data02/z5025317/Code_execution/P1_NARCliM_NC_to_CSV_CCRC_SS.py --lat $latitude --lon $longitude --varName 'potevpmean' --domain 'd02' --timestep 'DAY'
#1 The above code extracts time series from the full model ensemble over a single model grid cell (based on lat lon input) for the above variables of interest and stores into CSV files.
Example of output name = evspsblmean_35.76_150.25_NARCliM_summary.csv
#2 The "P1_NARCliM_plots_Windows" code takes these CSV files as input and creates a) a pdf wiht a number of time series and box plots and b) another CSV file containing the Deltas between present day, near and far future
for each model in the ensemble. Output: C:\Users\z5025317\WRL_Postdoc\Projects\Paper#1\Output\Nadgee\Nadgee_tasmax_NARCliM_ensemble_changes.csv
#3 The "P1_NARCliM_First_Pass_variab_deviation_plots" code takes those Delta CSV files as input and generates the future climate deviation plots that were originally developed by Duncan Rayner.
Run the code with different constellations of Estuary (study sites) and climate variables:
e.g. Clim_var_type = "tasmax*" # '*' will create pdf for all variables in folder
Present_Day_Clim_Var = 'MaxT' #MaxT, MinT, Rainfall, (the name for present day clim var refers to the name of the observed climate data that's used for the baseline period variability.
#!!!# Important!
#For present day variability data, only rainfall and temperature data actually correspond to the study sites. ET and Wind are taken from the existing project folder and hence, are for a Hunter weather station
#e.g. Terrigal_Wind and Terrigal_ET are actually Hunter in reality. This is because we don't have ET for other sites than Hunter at this stage.
##PROBLEM: Without changing anything, the P1_NARCliM_NC_to_CSV_CCRC_SS.py stopped working properly on the CCRC storm servers. It's not giving an error but loading the nc files with Dataset(nc) just takes unlimited time.
It used to take only a few seconds. NOT solved yet as of 7th of May 2018.
deactivate deactivate
conda env create --name EEenv -- file C:\Users\z5025317\WRL_Postdoc\Software\EE\ee-jupyter-examples-master\kilian_env conda env create --name EEenv -- file C:\Users\z5025317\WRL_Postdoc\Software\EE\ee-jupyter-examples-master\kilian_env

@ -18,22 +18,20 @@ from datetime import timedelta
from matplotlib.backends.backend_pdf import PdfPages from matplotlib.backends.backend_pdf import PdfPages
from ggplot import * from ggplot import *
matplotlib.style.use('ggplot') matplotlib.style.use('ggplot')
# #
# Set working direcotry (where postprocessed NARClIM data is located) # Set working direcotry (where postprocessed NARClIM data is located)
os.chdir('C:/Users/z5025317/WRL_Postdoc/Projects/Paper#1/') os.chdir('C:/Users/z5025317/WRL_Postdoc/Projects/Paper#1/')
# #
#
#####################################---------------------------------- #####################################----------------------------------
#set input parameters #set input parameters
Base_period_start = '1990-01-01' Base_period_start = '1990-01-01'
Base_period_end = '2080-01-01' #use last day that's not included in period as < is used for subsetting Base_period_end = '2080-01-01' #use last day that's not included in period as < is used for subsetting
Estuary = 'Terrigal' # 'Belongil' Estuary = 'Terrigal' # 'Belongil'
Clim_var_type = "pracc*|tasmax*" # '*' will create pdf for all variables in folder Clim_var_type = "wssmean*" # '*' will create pdf for all variables in folder "pracc*|tasmax*"
subset_ensemble = 'yes' # is yes, only the model with the lowest, median and max difference between present day and far future are selected subset_ensemble = 'yes' # is yes, only the model with the lowest, median and max difference between present day and far future are selected
plot_pdf = 'no' plot_pdf = 'no'
#####################################---------------------------------- #####################################----------------------------------
#
#set directory path for output files #set directory path for output files
output_directory = 'Output/'+ Estuary output_directory = 'Output/'+ Estuary
#output_directory = 'J:/Project wrl2016032/NARCLIM_Raw_Data/Extracted' #output_directory = 'J:/Project wrl2016032/NARCLIM_Raw_Data/Extracted'
@ -62,6 +60,8 @@ for clim_var_csv_path in Clim_Var_CSVs:
#substract a constant from all values to convert from kelvin to celcius (temp) #substract a constant from all values to convert from kelvin to celcius (temp)
if Clim_var_type == 'tasmean' or Clim_var_type == 'tasmax': if Clim_var_type == 'tasmean' or Clim_var_type == 'tasmax':
Full_df = Full_df.iloc[:,0:(Ncols_df-1)]-273.15 Full_df = Full_df.iloc[:,0:(Ncols_df-1)]-273.15
if Clim_var_type == 'evspsblmean' or Clim_var_type == 'potevpmean':
Full_df = Full_df.iloc[:,0:(Ncols_df-1)]*60*60*24
Fdf_1900_2080 = Full_df Fdf_1900_2080 = Full_df
#Subset the data to the minimum base period and above (used to set the lenght of the present day climate period) #Subset the data to the minimum base period and above (used to set the lenght of the present day climate period)

Loading…
Cancel
Save