From 342f4f3bc02a79ef66418d0c19af06e84fb318b5 Mon Sep 17 00:00:00 2001 From: Valentin Heimhuber Date: Fri, 8 Jun 2018 18:37:36 +1000 Subject: [PATCH] #new code for preparing the NARCLIM data for use in the rma_preprocessing.py code= it's fullly working! --- ...aring_Raw_NARCLIM_for_RMA_preprocessing.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/Preparing_Raw_NARCLIM_for_RMA_preprocessing.py diff --git a/07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/Preparing_Raw_NARCLIM_for_RMA_preprocessing.py b/07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/Preparing_Raw_NARCLIM_for_RMA_preprocessing.py new file mode 100644 index 0000000..e78f192 --- /dev/null +++ b/07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/Preparing_Raw_NARCLIM_for_RMA_preprocessing.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Jun 08 14:34:51 2018 + +@author: z5025317 +""" +#####################################---------------------------------- +#Last Updated - March 2018 +#@author: z5025317 Valentin Heimhuber +#code for generating P and ET time series as input to the HUNTER AWBM and RMA models +#Inputs: Uses CSV files that contain all 12 NARCLIM model runs time series for 1 grid cell created with: P1_NARCliM_NC_to_CSV_CCRC_SS.py +#####################################---------------------------------- + +#####################################---------------------------------- +#Load packages +#####################################---------------------------------- +import numpy as np +import os +import pandas as pd +import glob +import matplotlib +import matplotlib.pyplot as plt +from datetime import datetime +from datetime import timedelta +from matplotlib.backends.backend_pdf import PdfPages +from ggplot import * +matplotlib.style.use('ggplot') +# +# Set working direcotry (where postprocessed NARClIM data is located) +os.chdir('C:/Users/z5025317/OneDrive - UNSW/Hunter_CC_Modeling/07_Modelling/01_Input/BC_Generation/ClimateData/') +#set directory path for output files +output_directory = 'NARCLIM_Processed/Rainfall/' +#output_directory = 'J:/Project wrl2016032/NARCLIM_Raw_Data/Extracted' +if not os.path.exists(output_directory): + os.makedirs(output_directory) + print('-------------------------------------------') + print("output directory folder didn't exist and was generated") + print('-------------------------------------------') + print('-------------------') + +#####################################---------------------------------- +#set input parameters +Model_site = 'HUNTER' # 'Belongil' +Clim_var_type = "pracc" # '*' will create pdf for all variables in folder "pracc*|tasmax*" +Bias_Corrected = True +#####################################---------------------------------- + +Scenarios = ["NNRP_R1_1950", "NNRP_R2_1950", "NNRP_R3_1950", 'CCCMA3.1_R1_2020', 'CCCMA3.1_R2_2020', 'CCCMA3.1_R3_2020', + 'CSIRO-MK3.0_R1_2020', 'CSIRO-MK3.0_R2_2020', 'CSIRO-MK3.0_R3_2020', 'ECHAM5_R1_2020','ECHAM5_R2_2020', + 'ECHAM5_R3_2020','MIROC3.2_R1_2020', 'MIROC3.2_R2_2020', 'MIROC3.2_R3_2020'] + +#write the key plots to a single pdf document +pdf_out_file_name = Clim_var_type + '_summary_time_series_plots_NARCliM_summary_.pdf' +pdf_out_path = output_directory +'/' + pdf_out_file_name +#open pdf and add the plots +with PdfPages(pdf_out_path) as pdf: + for scenario in Scenarios: + #scenario = 'CCCMA3.1_R1_2020' + print scenario[:-5] + Catchment_directories = glob.glob('./NARCLIM/*') + Output_df = pd.DataFrame() + for Catchment_dir in Catchment_directories: + Filename = os.path.basename(os.path.normpath(Catchment_dir)) + Catchment = 'Catchment' + '_' + Filename.split('_', 2)[1] + if Bias_Corrected: + Catchment_dir = Catchment_dir + '/Bias_corrected' + if scenario[:4] == 'NNRP': + clim_var_csv_path = glob.glob(Catchment_dir + '/' + Clim_var_type + '_NNRP*') + Full_df = pd.read_csv(clim_var_csv_path[0], index_col=0, parse_dates = True) + Full_df.index = Full_df.index.to_period('D') + IVM_df = Full_df.filter(regex= scenario[:-5]) + IVM_df = IVM_df.iloc[:,:].loc[(IVM_df.index < '2010-01-01')] + #calculate daily ETo from the NARCLIM ET flux + if Clim_var_type == 'evspsblmean' or Clim_var_type == 'potevpmean': + IVM_df = IVM_df.multiply(86400) + IVM_df.columns = [Catchment] + Output_df = pd.concat([Output_df, IVM_df], axis=1) + else: + clim_var_csv_path = glob.glob(Catchment_dir + '/' + Clim_var_type + '_GCMS*') + Full_df = pd.read_csv(clim_var_csv_path[0], index_col=0, parse_dates = True) + Full_df.index = Full_df.index.to_period('D') + IVM_df = Full_df.filter(regex= scenario[:-5]) + IVM_df = IVM_df.iloc[:,:].loc[(IVM_df.index > '1989-12-31')] + if Clim_var_type == 'evspsblmean' or Clim_var_type == 'potevpmean': + IVM_df = IVM_df.multiply(86400) + IVM_df.columns = [scenario[:-5],scenario[:-5], scenario[:-5]] + dfa1 = pd.DataFrame(IVM_df.iloc[:,0]).dropna(axis=0) + dfa2 = pd.DataFrame(IVM_df.iloc[:,1]).dropna(axis=0) + dfa3 = pd.DataFrame(IVM_df.iloc[:,2]).dropna(axis=0) + IVM_df = pd.concat([dfa1, dfa2], axis=0) + IVM_df = pd.concat([IVM_df, dfa3], axis=0) + IVM_df.columns = [Catchment] + Output_df = pd.concat([Output_df, IVM_df], axis=1) + + Output_df.index.name = 'datetime' + Output_df_annual = Output_df.resample('A').sum() + Output_df_annual = Output_df_annual.replace(0, np.nan) + # time series plot annual ALL models + plt.title(Clim_var_type + ' - Time series - for all catchments') + Output_df_annual.plot(legend=False) + pdf.savefig(bbox_inches='tight', pad_inches=0.4) + plt.close() + + #export the pandas data frame as a CSV file within the output directory + out_file_name = scenario[:-5] + '_' + Clim_var_type + '_NARCliM.csv' + out_path = output_directory +'/' + out_file_name + Output_df.to_csv(out_path) \ No newline at end of file