chrisl
/
nsw-2016-storm-impact
2
1
Fork 0
Code Issues Pull Requests Releases Activity
You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
develop
master
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '6ffdd2611d'
${ noResults }
nsw-2016-storm-impact/notebooks/07_evaluate_model_performan...

493 lines
16 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Evaluate prediction metrics \n",
"- This notebook is used to check out each of our storm impact prediction models performed in comparison to our observed storm impacts."
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Setup notebook\n",
"Import our required packages and set default plotting options."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# Enable autoreloading of our modules. \n",
"# Most of the code will be located in the /src/ folder, \n",
"# and then called from the notebook.\n",
"%matplotlib inline\n",
"%reload_ext autoreload\n",
"%autoreload"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"from IPython.core.debugger import set_trace\n",
"\n",
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"import decimal\n",
"import plotly\n",
"import plotly.graph_objs as go\n",
"import plotly.plotly as py\n",
"import plotly.tools as tls\n",
"import plotly.figure_factory as ff\n",
"from plotly import tools\n",
"import plotly.io as pio\n",
"from scipy import stats\n",
"import math\n",
"import matplotlib\n",
"from matplotlib import cm\n",
"import colorlover as cl\n",
"from tqdm import tqdm_notebook\n",
"from ipywidgets import widgets, Output\n",
"from IPython.display import display, clear_output, Image, HTML\n",
"from scipy import stats\n",
"from sklearn.metrics import confusion_matrix\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib.lines import Line2D\n",
"from cycler import cycler\n",
"from scipy.interpolate import interp1d\n",
"from pandas.api.types import CategoricalDtype"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"# Matplot lib default settings\n",
"plt.rcParams[\"figure.figsize\"] = (10,6)\n",
"plt.rcParams['axes.grid']=True\n",
"plt.rcParams['grid.alpha'] = 0.5\n",
"plt.rcParams['grid.color'] = \"grey\"\n",
"plt.rcParams['grid.linestyle'] = \"--\"\n",
"plt.rcParams['axes.grid']=True\n",
"\n",
"# https://stackoverflow.com/a/20709149\n",
"matplotlib.rcParams['text.usetex'] = True\n",
"\n",
"matplotlib.rcParams['text.latex.preamble'] = [\n",
" r'\\usepackage{siunitx}', # i need upright \\micro symbols, but you need...\n",
" r'\\sisetup{detect-all}', # ...this to force siunitx to actually use your fonts\n",
" r'\\usepackage{helvet}', # set the normal font here\n",
" r'\\usepackage{amsmath}',\n",
" r'\\usepackage{sansmath}', # load up the sansmath so that math -> helvet\n",
" r'\\sansmath', # <- tricky! -- gotta actually tell tex to use!\n",
"] "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Import data\n",
"Import our data from the `./data/interim/` folder and load it into pandas dataframes. "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def df_from_csv(csv, index_col, data_folder='../data/interim'):\n",
" print('Importing {}'.format(csv))\n",
" return pd.read_csv(os.path.join(data_folder,csv), index_col=index_col)\n",
"\n",
"df_waves = df_from_csv('waves.csv', index_col=[0, 1])\n",
"df_tides = df_from_csv('tides.csv', index_col=[0, 1])\n",
"df_profiles = df_from_csv('profiles.csv', index_col=[0, 1, 2])\n",
"df_sites = df_from_csv('sites.csv', index_col=[0])\n",
"df_profile_features_crest_toes = df_from_csv('profile_features_crest_toes.csv', index_col=[0,1])\n",
"\n",
"# Note that the forecasted data sets should be in the same order for impacts and twls\n",
"impacts = {\n",
" 'forecasted': {\n",
" 'foreshore_slope_sto06': df_from_csv('impacts_forecasted_foreshore_slope_sto06.csv',index_col=[0]),\n",
" 'mean_slope_hol86': df_from_csv('impacts_forecasted_mean_slope_hol86.csv',index_col=[0]),\n",
" 'mean_slope_nie91': df_from_csv('impacts_forecasted_mean_slope_nie91.csv',index_col=[0]),\n",
" 'mean_slope_pow18': df_from_csv('impacts_forecasted_mean_slope_pow18.csv',index_col=[0]),\n",
" 'mean_slope_sto06': df_from_csv('impacts_forecasted_mean_slope_sto06.csv',index_col=[0]),\n",
" },\n",
" 'observed': df_from_csv('impacts_observed.csv', index_col=[0])\n",
" }\n",
"\n",
"\n",
"twls = {\n",
" 'forecasted': {\n",
" 'foreshore_slope_sto06': df_from_csv('twl_foreshore_slope_sto06.csv',index_col=[0,1]),\n",
" 'mean_slope_hol86': df_from_csv('twl_mean_slope_hol86.csv',index_col=[0,1]),\n",
" 'mean_slope_nie91': df_from_csv('twl_mean_slope_nie91.csv',index_col=[0,1]),\n",
" 'mean_slope_pow18': df_from_csv('twl_mean_slope_pow18.csv',index_col=[0,1]),\n",
" 'mean_slope_sto06': df_from_csv('twl_mean_slope_sto06.csv',index_col=[0,1]),\n",
" }\n",
"}\n",
"print('Done!')"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"## Generate longshore plots for each beach"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"beach = 'NARRA'\n",
"\n",
"df_obs_impacts = impacts['observed'].loc[impacts['observed'].index.str.\n",
" contains(beach)]\n",
"\n",
"# Get index for each site on the beach\n",
"n = [x for x in range(len(df_obs_impacts))][::-1]\n",
"n_sites = [x for x in df_obs_impacts.index][::-1]\n",
"\n",
"# Convert storm regimes to categorical datatype\n",
"cat_type = CategoricalDtype(\n",
" categories=['swash', 'collision', 'overwash', 'inundation'], ordered=True)\n",
"df_obs_impacts.storm_regime = df_obs_impacts.storm_regime.astype(cat_type)\n",
"\n",
"# Create figure\n",
"f, (ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8) = plt.subplots(\n",
" 1,\n",
" 8,\n",
" sharey=True,\n",
" figsize=(18, 8),\n",
" gridspec_kw={'width_ratios': [4, 4, 2, 2, 2, 2, 2, 2]})\n",
"\n",
"# ax1: Impacts\n",
"\n",
"# Define colors for storm regime\n",
"cmap = {'swash': '#1a9850', 'collision': '#fee08b', 'overwash': '#d73027'}\n",
"\n",
"# Common marker style\n",
"marker_style = {\n",
" 's': 60,\n",
" 'linewidths': 0.7,\n",
" 'alpha': 1,\n",
" 'edgecolors': 'k',\n",
" 'marker': 'o',\n",
"}\n",
"\n",
"# Plot observed impacts\n",
"colors = [cmap.get(x) for x in df_obs_impacts.storm_regime]\n",
"colors = ['#d73027' if c is None else c for c in colors]\n",
"ax1.scatter([0 for x in n], n, color=colors, **marker_style)\n",
"\n",
"# Plot model impacts\n",
"for i, model in enumerate(impacts['forecasted']):\n",
"\n",
" # Only get model results for this beach\n",
" df_model = impacts['forecasted'][model].loc[impacts['forecasted'][model].\n",
" index.str.contains(beach)]\n",
"\n",
" # Recast storm regimes as categorical data\n",
" df_model.storm_regime = df_model.storm_regime.astype(cat_type)\n",
"\n",
" # Assign colors\n",
" colors = [cmap.get(x) for x in df_model.storm_regime]\n",
" colors = ['#aaaaaa' if c is None else c for c in colors]\n",
" ax1.scatter([i + 1 for x in n], n, color=colors, **marker_style)\n",
"\n",
"# Add model names to each impact on x axis\n",
"ax1.set_xticks(range(len(impacts['forecasted']) + 1))\n",
"ax1.set_xticklabels(['observed'] +\n",
" [x.replace('_', '\\_') for x in impacts['forecasted']])\n",
"ax1.xaxis.set_tick_params(rotation=90)\n",
"\n",
"# Add title\n",
"ax1.set_title('Storm regime')\n",
"\n",
"# Create custom legend\n",
"legend_elements = [\n",
" Line2D([0], [0],\n",
" marker='o',\n",
" color='w',\n",
" label='Swash',\n",
" markerfacecolor='#1a9850',\n",
" markersize=8,\n",
" markeredgewidth=1.0,\n",
" markeredgecolor='k'),\n",
" Line2D([0], [0],\n",
" marker='o',\n",
" color='w',\n",
" label='Collision',\n",
" markerfacecolor='#fee08b',\n",
" markersize=8,\n",
" markeredgewidth=1.0,\n",
" markeredgecolor='k'),\n",
" Line2D([0], [0],\n",
" marker='o',\n",
" color='w',\n",
" label='Overwash',\n",
" markerfacecolor='#d73027',\n",
" markersize=8,\n",
" markeredgewidth=1.0,\n",
" markeredgecolor='k'),\n",
"]\n",
"ax1.legend(\n",
" handles=legend_elements, loc='lower center', bbox_to_anchor=(0.5, 1.1))\n",
"\n",
"# Replace yticks with site_ids\n",
"yticks = ax1.get_yticks().tolist()\n",
"yticks = [n_sites[int(y)] if 0 <= y <= len(n_sites) else y for y in yticks]\n",
"ax1.set_yticklabels(yticks)\n",
"\n",
"# ax2: elevations\n",
"\n",
"# Dune elevations\n",
"df_feats = df_profile_features_crest_toes.xs(['prestorm'],\n",
" level=['profile_type'])\n",
"df_feats = df_feats.loc[df_feats.index.str.contains(beach)]\n",
"\n",
"ax2.plot(df_feats.dune_crest_z, n, color='#fdae61')\n",
"ax2.plot(df_feats.dune_toe_z, n, color='#fdae61')\n",
"ax2.fill_betweenx(\n",
" n,\n",
" df_feats.dune_toe_z,\n",
" df_feats.dune_crest_z,\n",
" alpha=0.2,\n",
" color='#fdae61',\n",
" label='$D_{low}$ to $D_{high}$')\n",
"\n",
"model_colors = [\n",
" '#1f78b4',\n",
" '#33a02c',\n",
" '#e31a1c',\n",
" '#6a3d9a',\n",
" '#a6cee3',\n",
" '#b2df8a',\n",
" '#fb9a99',\n",
" '#cab2d6',\n",
" '#ffff99',\n",
" ]\n",
"\n",
"# Define colors to cycle through for our R_high\n",
"ax2.set_prop_cycle(cycler('color', model_colors))\n",
"\n",
"# Plot R_high values\n",
"for model in impacts['forecasted']:\n",
"\n",
" # Only get model results for this beach\n",
" df_model = impacts['forecasted'][model].loc[impacts['forecasted'][model].\n",
" index.str.contains(beach)]\n",
"\n",
" # Recast storm regimes as categorical data\n",
" ax2.plot(df_model.R_high, n, label=model.replace('_', '\\_'))\n",
"\n",
"# Set title, legend and labels\n",
"ax2.set_title('TWL \\& dune\\nelevations')\n",
"ax2.legend(loc='lower center', bbox_to_anchor=(0.5, 1.1))\n",
"ax2.set_xlabel('Elevation (m AHD)')\n",
"\n",
"\n",
"# ax3: Plot R_high - D_low\n",
"\n",
"# Define colors to cycle through for our R_high\n",
"ax3.set_prop_cycle(cycler('color', model_colors))\n",
"\n",
"# Plot R_high values\n",
"for model in impacts['forecasted']:\n",
" \n",
" df_model = impacts['forecasted'][model].loc[impacts['forecasted'][model].\n",
" index.str.contains(beach)]\n",
" # R_high - D_low\n",
" ax3.plot(df_model.R_high - df_feats.dune_toe_z, n, label=model.replace('_', '\\_'))\n",
"\n",
"ax3.axvline(x=0,color='black',linestyle=':')\n",
"ax3.set_title('$R_{high}$ - $D_{low}$')\n",
"ax3.set_xlabel('Height (m)')\n",
"ax3.set_xlim([-2,2])\n",
"\n",
"\n",
"\n",
"# Define colors to cycle through for our R2\n",
"ax4.set_prop_cycle(cycler('color', model_colors))\n",
"\n",
"# R_high - D_low\n",
"for model in impacts['forecasted']:\n",
" df_R2 = impacts['forecasted'][model].merge(twls['forecasted'][model],on=['site_id','datetime'])\n",
" df_R2 = df_R2.loc[df_R2.index.str.contains(beach)]\n",
" ax4.plot(df_R2.R2, n, label=model.replace('_', '\\_'))\n",
"\n",
"ax4.set_title(r'$R_{2\\%}$')\n",
"ax4.set_xlabel('Height (m)')\n",
"\n",
"# Need to chose a model to extract environmental parameters at maximum R_high time\n",
"model = 'mean_slope_sto06'\n",
"df_beach = impacts['forecasted'][model].merge(twls['forecasted'][model], on=['site_id','datetime'])\n",
"df_beach = df_beach.loc[df_beach.index.str.contains(beach)]\n",
"\n",
"# Wave height, wave period, beach slope\n",
"ax5.plot(df_beach.beta, n,color='#4daf4a')\n",
"ax5.set_title(r'$\\beta$')\n",
"ax5.set_xlabel('Mean prestorm\\nbeach slope')\n",
"ax5.set_xlim([0,0.15])\n",
"\n",
"ax6.plot(df_beach.Hs0, n,color='#999999')\n",
"ax6.set_title('$H_{s0}$')\n",
"ax6.set_xlabel('Sig. wave height (m)')\n",
"ax6.set_xlim([3,5])\n",
"\n",
"ax7.plot(df_beach.Tp, n,color='#999999')\n",
"ax7.set_title('$T_{p}$')\n",
"ax7.set_xlabel('Peak wave period (s)')\n",
"ax7.set_xlim([8,14])\n",
"\n",
"ax8.plot(df_beach.tide, n,color='#999999')\n",
"ax8.set_title('Tide \\& surge')\n",
"ax8.set_xlabel('Elevation (m AHD)')\n",
"ax8.set_xlim([0,2])\n",
"\n",
"\n",
"plt.tight_layout()\n",
"f.subplots_adjust(top=0.88)\n",
"f.suptitle(beach)\n",
"\n",
"# # Print to figure\n",
"plt.savefig('07_{}.png'.format(beach), dpi=600, bbox_inches='tight')\n",
"\n",
"plt.show()\n",
"plt.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Generate classification reports for each model\n",
"Use sklearn metrics to generate classification reports for each forecasting model."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sklearn.metrics\n",
"\n",
"# Get observed impacts\n",
"df_obs = impacts['observed']\n",
"\n",
"# Convert storm regimes to categorical datatype\n",
"cat_type = CategoricalDtype(\n",
" categories=['swash', 'collision', 'overwash', 'inundation'], ordered=True)\n",
"df_obs.storm_regime = df_obs_impacts.storm_regime.astype(cat_type)\n",
"\n",
"for model in impacts['forecasted']:\n",
" df_for = impacts['forecasted'][model]\n",
" df_for.storm_regime = df_for.storm_regime.astype(cat_type)\n",
"\n",
" m = sklearn.metrics.classification_report(\n",
" df_obs.storm_regime.astype(cat_type).cat.codes.values,\n",
" df_for.storm_regime.astype(cat_type).cat.codes.values,\n",
" labels=[0, 1, 2, 3],\n",
" target_names=['swash', 'collision', 'overwash', 'inundation'])\n",
" print(model)\n",
" print(m)\n",
" print()\n",
" "
]
}
],
"metadata": {
"hide_input": false,
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink