diff --git a/Analysis/Code/NARCLIM_Download/R_Preparing_BASH_script_for_NARCLIM_batch_download.R b/Analysis/Code/NARCLIM_Download/R_Preparing_BASH_script_for_NARCLIM_batch_download.R new file mode 100644 index 0000000..98c28ab --- /dev/null +++ b/Analysis/Code/NARCLIM_Download/R_Preparing_BASH_script_for_NARCLIM_batch_download.R @@ -0,0 +1,59 @@ +#code for preparing a text file with BASH code for batch download of NARCLIM data for the HUNTER WQ modeling of +#future climate scenarios + +#NARCLIM Variables +#evspsblmean water_evaporation flux (actual ET) long_name: Surface evaporation standard_name: water_evaporation_flux units: kg m-2 s-1 +#tasmean mean near surface temperature +#pracc precipitation daily precipitation sum (sum of convective prcacc and stratiform prncacc precip) + +Clim_Var <- 'evspsblmean' +Datatype <- 'T_GCMS' #T_GCMS for GCM forcing, T_NNRP for reanalysis (only 1950-2009) +Biasboolean <- 'False' #use bias corrected data? + +Directory <- 'C:/Users/z5025317/OneDrive - UNSW/WRL_Postdoc_Manual_Backup/WRL_Postdoc/Projects/Paper#1/Data/NARCLIM_Site_CSVs' +Filename <- 'NARCLIM_Point_Sites.csv' + +#Load CSV with location names and lat lon coordinates +Location.df <- data.frame(read.csv(paste(Directory, Filename, sep=""), header=T)) + +#create empty vector for storing the command line text and open file +Vector.for.command.line.txt <- c() +Vector.for.command.line.txt <- c(Vector.for.command.line.txt, "module load python") +text1 <- c(paste("Datatype='",Datatype,"'", sep=""), + paste("Bias_corrected='",Biasboolean,"'", sep=""), paste("ClimVarName='",Clim_Var,"'", sep="")) +Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text1) +for (i in 1:(length(Location.df$Name))){ + #name<-as.character(Location.df$Name[i]) + #name<-gsub('([[:punct:]])|\\s+','_',name) + if(i<10){ + name<-paste('Catchment_0', as.character(i), sep="") + }else{ + name<-paste('Catchment_', as.character(i), sep="") + } + latitude=round(as.numeric(Location.df$Lat[i]),3) + longitude=round(as.numeric(Location.df$Long[i]),3) + text <- c(paste("latitude=",latitude,"", sep=""), paste("longitude=",longitude,"", sep=""), +paste("name='",name,"'", sep=""), +"python /srv/ccrc/data02/z5025317/Code_execution/\\ +P1_NARCliM_NC_to_CSV_CCRC_SS.py \\ + --lat $latitude --lon $longitude --varName $ClimVarName --domain 'd02' --timestep \\ + 'DAY' --LocationName $name --Datatype $Datatype --BiasBool $Bias_corrected") + Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text) + if(i==10|i==20|i==31){ + Vector.for.command.line.txt <- c(Vector.for.command.line.txt, " ") + text.file.name <- paste('C:/Users/z5025317/OneDrive - UNSW/Hunter_CC_Modeling/07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/',Clim_Var, "_", Datatype, "_", Biasboolean,substring(as.character(i), 1,1), ".txt", sep="") + #open and fill text file + fileConn <- file(text.file.name) + writeLines(Vector.for.command.line.txt, fileConn) + close(fileConn) + # + if(i==10|i==20){ + Vector.for.command.line.txt <- c() + Vector.for.command.line.txt <- c(Vector.for.command.line.txt, "module load python") + text1 <- c(paste("Datatype='",Datatype,"'", sep=""), + paste("Bias_corrected='",Biasboolean,"'", sep=""), paste("ClimVarName='",Clim_Var,"'", sep="")) + Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text1) + } + } + } + diff --git a/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS.py b/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS.py index 5914cbe..f62f4ea 100644 --- a/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS.py +++ b/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS.py @@ -13,10 +13,7 @@ import argparse import time # # Set working direcotry (where postprocessed NARClIM data is located) -os.chdir('/srv/ccrc/data30/z3393020/NARCliM/postprocess/') -# -#Choose Type of NARCLIM data ('T_NNRP', 'T_GCMS') -Data_Type = 'T_NNRP' +os.chdir('/srv/ccrc/data30/z3393020/NARCliM/postprocess/') # #User input for location and variable type - from command line if __name__ == "__main__": @@ -43,15 +40,20 @@ if __name__ == "__main__": Bias_Correction_BOOL = args.BiasBool print("Extracting all NARCLIM time series for variable: ", Clim_var_type, " for lat lon: ", mylat, mylon, Location, "domain", NC_Domain, " timestep ", Timestep, " Datatype ", Data_Type, " biascorrected? ", Bias_Correction_BOOL) +lat_equal_len_string="%.3f" % abs(mylat) +lon_equal_len_string= "%.3f" % mylon + if Bias_Correction_BOOL == 'False': #set directory path for output files - output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '/' + output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' + lat_equal_len_string + '_' + lon_equal_len_string + '/' #output_directory = 'J:\Project wrl2016032\NARCLIM_Raw_Data\Extracted' + print '---------------------------------------------------------' if not os.path.exists(output_directory): os.makedirs(output_directory) print("output directory folder didn't exist and was generated here:") print(output_directory) - + print '---------------------------------------------------------' + # #time.sleep(10) #set up the loop variables for interrogating the entire NARCLIM raw data NC_Periods = ('1990-2009','2020-2039','2060-2079') @@ -137,14 +139,11 @@ if Bias_Correction_BOOL == 'False': Full_df.loc[(Full_df.index > '1990-01-01') & (Full_df.index < '2010-01-01'), 'period']= '1990-2009' Full_df.loc[(Full_df.index > '2020-01-01') & (Full_df.index < '2040-01-01'), 'period']= '2020-2039' Full_df.loc[(Full_df.index > '2060-01-01') & (Full_df.index < '2080-01-01'), 'period']= '2060-2079' - #export the pandas data frame as a CSV file within the output directory - out_file_name = Clim_var_type + '_'+ Data_Type[2:] + '_' + Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '_NARCliM_summary.csv' - out_path = output_directory +'/' + out_file_name - Full_df.to_csv(out_path) # if Bias_Correction_BOOL == 'True': + os.chdir('/srv/ccrc/data30/z3393020/NARCliM/Bias_corrected/') #set directory path for output files - output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '/Bias_corrected/' + output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' + lat_equal_len_string + '_' + lon_equal_len_string + '/Bias_corrected/' #output_directory = 'J:\Project wrl2016032\NARCLIM_Raw_Data\Extracted' if not os.path.exists(output_directory): os.makedirs(output_directory) @@ -228,11 +227,7 @@ if Bias_Correction_BOOL == 'True': Full_df = pd.concat([Full_df, GCM_df], axis=1) GCM_df = pd.DataFrame() Full_df = Full_df.sort_index(axis=0, ascending=True) - #adding a column with the NARCLIM decade - #Full_df.loc[(Full_df.index > '1990-01-01') & (Full_df.index < '2009-01-01'), 'period']= '1990-2009' - #Full_df.loc[(Full_df.index > '2020-01-01') & (Full_df.index < '2039-01-01'), 'period']= '2020-2039' - #Full_df.loc[(Full_df.index > '2060-01-01') & (Full_df.index < '2079-01-01'), 'period']= '2060-2079' - #export the pandas data frame as a CSV file within the output directory - out_file_name = Clim_var_type + '_'+ Data_Type[2:] + '_' + Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '_NARCliM_summary.csv' - out_path = output_directory +'/' + out_file_name - Full_df.to_csv(out_path) \ No newline at end of file +#export the pandas data frame as a CSV file within the output directory +out_file_name = Clim_var_type + '_'+ Data_Type[2:] + '_' + Location + '_' + lat_equal_len_string + '_' + lon_equal_len_string + '_NARCliM_summary.csv' +out_path = output_directory +'/' + out_file_name +Full_df.to_csv(out_path) \ No newline at end of file diff --git a/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS_BASH_script_readme.txt b/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS_BASH_script_readme.txt index 0f2e9e8..6f9fb41 100644 --- a/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS_BASH_script_readme.txt +++ b/Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS_BASH_script_readme.txt @@ -74,7 +74,7 @@ Present_Day_Clim_Var = 'MaxT' #MaxT, MinT, Rainfall, (the name for present da ##PROBLEM: Without changing anything, the P1_NARCliM_NC_to_CSV_CCRC_SS.py stopped working properly on the CCRC storm servers. It's not giving an error but loading the nc files with Dataset(nc) just takes unlimited time. -It used to take only a few seconds. NOT solved yet as of 7th of May 2018. +It used to take only a few seconds. NOT solved yet as of 7th of May 2018.### This was solved for the /postprocessed folder at the end of May 2018 but the problem persits with the /bias_corrected/ data folder. running a simple netcdf info script python /srv/ccrc/data02/z5025317/Code_execution/P1_Basic_NETCDF_Interrogation.py