#added a few additions to the NARCLIM NC code and added an R code for

generating BASH code for automating the download
for a number of sites and clim_vars

Analysis/Code/NARCLIM_Download/R_Preparing_BASH_script_for_NARCLIM_batch_download.R

@@ -0,0 +1,59 @@
+#code for preparing a text file with BASH code for batch download of NARCLIM data for the HUNTER WQ modeling of 
+#future climate scenarios 
+
+#NARCLIM Variables
+#evspsblmean   	water_evaporation flux (actual ET) long_name: Surface evaporation standard_name: water_evaporation_flux units: kg m-2 s-1
+#tasmean      mean near surface temperature
+#pracc 		precipitation daily precipitation sum (sum of convective prcacc and stratiform prncacc precip) 
+
+Clim_Var <- 'evspsblmean'
+Datatype <- 'T_GCMS'  #T_GCMS for GCM forcing, T_NNRP for reanalysis (only 1950-2009)
+Biasboolean <- 'False' #use bias corrected data?
+
+Directory <- 'C:/Users/z5025317/OneDrive - UNSW/WRL_Postdoc_Manual_Backup/WRL_Postdoc/Projects/Paper#1/Data/NARCLIM_Site_CSVs'   
+Filename <- 'NARCLIM_Point_Sites.csv'
+
+#Load CSV with location names and lat lon coordinates
+Location.df <- data.frame(read.csv(paste(Directory, Filename, sep=""), header=T))
+
+#create empty vector for storing the command line text and open file
+Vector.for.command.line.txt <- c()
+Vector.for.command.line.txt <- c(Vector.for.command.line.txt, "module load python")
+text1 <- c(paste("Datatype='",Datatype,"'", sep=""),
+           paste("Bias_corrected='",Biasboolean,"'", sep=""), paste("ClimVarName='",Clim_Var,"'", sep=""))
+Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text1)
+for (i in 1:(length(Location.df$Name))){
+  #name<-as.character(Location.df$Name[i])
+  #name<-gsub('([[:punct:]])|\\s+','_',name)
+  if(i<10){
+    name<-paste('Catchment_0', as.character(i), sep="")
+  }else{
+    name<-paste('Catchment_', as.character(i), sep="")
+  }
+  latitude=round(as.numeric(Location.df$Lat[i]),3)
+  longitude=round(as.numeric(Location.df$Long[i]),3)
+  text <- c(paste("latitude=",latitude,"", sep=""), paste("longitude=",longitude,"", sep=""),
+paste("name='",name,"'", sep=""),
+"python /srv/ccrc/data02/z5025317/Code_execution/\\
+P1_NARCliM_NC_to_CSV_CCRC_SS.py \\
+  --lat $latitude --lon $longitude --varName $ClimVarName --domain 'd02' --timestep \\
+  'DAY' --LocationName $name --Datatype $Datatype --BiasBool $Bias_corrected")
+  Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text)
+  if(i==10|i==20|i==31){
+    Vector.for.command.line.txt <- c(Vector.for.command.line.txt, " ")
+    text.file.name <- paste('C:/Users/z5025317/OneDrive - UNSW/Hunter_CC_Modeling/07_Modelling/01_Input/BC_Generation/Code/NARCLIM_Download_and_Processing/',Clim_Var, "_", Datatype, "_", Biasboolean,substring(as.character(i), 1,1), ".txt", sep="")
+    #open and fill text file
+    fileConn <- file(text.file.name)
+    writeLines(Vector.for.command.line.txt, fileConn)
+    close(fileConn)
+    #
+    if(i==10|i==20){
+      Vector.for.command.line.txt <- c()
+      Vector.for.command.line.txt <- c(Vector.for.command.line.txt, "module load python")
+      text1 <- c(paste("Datatype='",Datatype,"'", sep=""),
+                 paste("Bias_corrected='",Biasboolean,"'", sep=""), paste("ClimVarName='",Clim_Var,"'", sep=""))
+      Vector.for.command.line.txt <- c(Vector.for.command.line.txt, text1)
+      }
+    }
+  }
+

Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS.py

@@ -15,9 +15,6 @@ import time
 # Set working direcotry (where postprocessed NARClIM data is located)
 os.chdir('/srv/ccrc/data30/z3393020/NARCliM/postprocess/') 
 #
-#Choose Type of NARCLIM data ('T_NNRP', 'T_GCMS')
-Data_Type = 'T_NNRP' 
-#
 #User input for location and variable type - from command line
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -43,15 +40,20 @@ if __name__ == "__main__":
     Bias_Correction_BOOL = args.BiasBool
     print("Extracting all NARCLIM time series for variable: ", Clim_var_type, " for lat lon: ", mylat, mylon, Location, "domain", NC_Domain, " timestep ", Timestep, " Datatype ", Data_Type, " biascorrected? ", Bias_Correction_BOOL)
 
+lat_equal_len_string="%.3f" % abs(mylat)
+lon_equal_len_string= "%.3f" % mylon
+
 if Bias_Correction_BOOL == 'False':
     #set directory path for output files
-    output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' +  str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '/'
+    output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' +  lat_equal_len_string + '_' + lon_equal_len_string + '/'
     #output_directory = 'J:\Project wrl2016032\NARCLIM_Raw_Data\Extracted'
+    print '---------------------------------------------------------'
     if not os.path.exists(output_directory):
         os.makedirs(output_directory)
         print("output directory folder didn't exist and was generated here:")
         print(output_directory)
-    
+    print '---------------------------------------------------------'
+    #
     #time.sleep(10) 
     #set up the loop variables for interrogating the entire NARCLIM raw data
     NC_Periods = ('1990-2009','2020-2039','2060-2079')
@@ -137,14 +139,11 @@ if Bias_Correction_BOOL == 'False':
     Full_df.loc[(Full_df.index > '1990-01-01') & (Full_df.index < '2010-01-01'), 'period']= '1990-2009'
     Full_df.loc[(Full_df.index > '2020-01-01') & (Full_df.index < '2040-01-01'), 'period']= '2020-2039'
     Full_df.loc[(Full_df.index > '2060-01-01') & (Full_df.index < '2080-01-01'), 'period']= '2060-2079'
-    #export the pandas data frame as a CSV file within the output directory
-    out_file_name = Clim_var_type + '_'+  Data_Type[2:] + '_' + Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '_NARCliM_summary.csv'
-    out_path = output_directory +'/' + out_file_name
-    Full_df.to_csv(out_path)
 #
 if Bias_Correction_BOOL == 'True':
+    os.chdir('/srv/ccrc/data30/z3393020/NARCliM/Bias_corrected/')
     #set directory path for output files
-    output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' +  str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '/Bias_corrected/'
+    output_directory = '/srv/ccrc/data02/z5025317/NARCliM_out/'+ Location + '_' +  lat_equal_len_string + '_' + lon_equal_len_string + '/Bias_corrected/'
     #output_directory = 'J:\Project wrl2016032\NARCLIM_Raw_Data\Extracted'
     if not os.path.exists(output_directory):
         os.makedirs(output_directory)
@@ -228,11 +227,7 @@ if Bias_Correction_BOOL == 'True':
         Full_df = pd.concat([Full_df, GCM_df], axis=1)
         GCM_df = pd.DataFrame()
     Full_df = Full_df.sort_index(axis=0, ascending=True)
-    #adding a column with the NARCLIM decade
+#export the pandas data frame as a CSV file within the output directory
-    #Full_df.loc[(Full_df.index > '1990-01-01') & (Full_df.index < '2009-01-01'), 'period']= '1990-2009'
+out_file_name = Clim_var_type + '_'+  Data_Type[2:] + '_' + Location + '_' + lat_equal_len_string + '_' + lon_equal_len_string + '_NARCliM_summary.csv'
-    #Full_df.loc[(Full_df.index > '2020-01-01') & (Full_df.index < '2039-01-01'), 'period']= '2020-2039'
+out_path = output_directory +'/' + out_file_name
-    #Full_df.loc[(Full_df.index > '2060-01-01') & (Full_df.index < '2079-01-01'), 'period']= '2060-2079'
+Full_df.to_csv(out_path)
-    #export the pandas data frame as a CSV file within the output directory
-    out_file_name = Clim_var_type +  '_'+  Data_Type[2:] + '_' + Location + '_' + str(abs(round(mylat,3))) + '_' + str(round(mylon, 3)) + '_NARCliM_summary.csv'
-    out_path = output_directory +'/' + out_file_name
-    Full_df.to_csv(out_path)

Analysis/Code/P1_NARCliM_NC_to_CSV_CCRC_SS_BASH_script_readme.txt

@@ -74,7 +74,7 @@ Present_Day_Clim_Var = 'MaxT'    #MaxT, MinT, Rainfall, (the name for present da
 
 
 ##PROBLEM: Without changing anything, the P1_NARCliM_NC_to_CSV_CCRC_SS.py stopped working properly on the CCRC storm servers. It's not giving an error but loading the nc files with Dataset(nc) just takes unlimited time.
-It used to take only a few seconds. NOT solved yet as of 7th of May 2018.
+It used to take only a few seconds. NOT solved yet as of 7th of May 2018.### This was solved for the /postprocessed folder at the end of May 2018 but the problem persits with the /bias_corrected/ data folder. 
 
 running a simple netcdf info script
 python /srv/ccrc/data02/z5025317/Code_execution/P1_Basic_NETCDF_Interrogation.py