nsw-2016-storm-impact/notebooks/01_exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data exploration\n",
    "This notebook provides an example how the data has been loaded and accessed for further analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:16.641947Z",
     "start_time": "2018-11-27T03:27:15.791353Z"
    }
   },
   "outputs": [],
   "source": [
    "# Enable autoreloading of our modules. \n",
    "# Most of the code will be located in the /src/ folder, \n",
    "# and then called from the notebook.\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:16.655952Z",
     "start_time": "2018-11-27T03:27:16.642938Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import plotly\n",
    "import plotly.graph_objs as go\n",
    "import plotly.plotly as py\n",
    "import plotly.tools as tls\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "import matplotlib\n",
    "from matplotlib import cm\n",
    "import colorlover as cl\n",
    "\n",
    "from ipywidgets import widgets, Output\n",
    "from IPython.display import display, clear_output, Image, HTML\n",
    "\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import our data into pandas Dataframes for the analysis. Data files are `.csv` files which are stored in the `./data/interim/` folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:31.268696Z",
     "start_time": "2018-11-27T03:27:16.657959Z"
    },
    "pixiedust": {
     "displayParams": {}
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\z5189959\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py:472: FutureWarning:\n",
      "\n",
      "elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def df_from_csv(csv, index_col, data_folder='../data/interim'):\n",
    "    return pd.read_csv(os.path.join(data_folder,csv), index_col=index_col)\n",
    "\n",
    "df_waves = df_from_csv('waves.csv', index_col=[0, 1])\n",
    "df_tides = df_from_csv('tides.csv', index_col=[0, 1])\n",
    "df_profiles = df_from_csv('profiles.csv', index_col=[0, 1, 2])\n",
    "df_sites = df_from_csv('sites.csv', index_col=[0])\n",
    "df_profile_features = df_from_csv('profile_features.csv', index_col=[0])\n",
    "\n",
    "# Note that the forecasted data sets should be in the same order for impacts and twls\n",
    "impacts = {\n",
    "    'forecasted': {\n",
    "        'foreshore_slope_sto06': df_from_csv('impacts_forecasted_foreshore_slope_sto06.csv', index_col=[0]),\n",
    "        'mean_slope_sto06': df_from_csv('impacts_forecasted_mean_slope_sto06.csv', index_col=[0]),\n",
    "        },\n",
    "    'observed': df_from_csv('impacts_observed.csv', index_col=[0])\n",
    "    }\n",
    "\n",
    "\n",
    "twls = {\n",
    "    'forecasted': {\n",
    "        'foreshore_slope_sto06': df_from_csv('twl_foreshore_slope_sto06.csv', index_col=[0, 1]),\n",
    "        'mean_slope_sto06':df_from_csv('twl_mean_slope_sto06.csv', index_col=[0, 1]),\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following interactive data explorer displays information on a per `site_id` basis. It can be used to examine pre/post storm cross-sections, water level time series and observed/predicted storm impacts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:34.422864Z",
     "start_time": "2018-11-27T03:27:31.278426Z"
    },
    "code_folding": [
     408
    ],
    "hide_input": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80589953e2e04fecb2a0f3add40b3d5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "VBox(children=(VBox(children=(HTML(value='<b>Filter by observed and predicted impacts:</b>'), HBox(children=(VBox(children=(HTML(value='Observed Impacts'), SelectMultiple(index=(0, 1), options=('swash', 'collision'), value=('swash', 'collision')))), VBox(children=(HTML(value='Forecasted: foreshore_slope_sto06'), SelectMultiple(index=(0, 1, 2), options=('swash', 'collision', 'overwash'), value=('swash', 'collision', 'overwash')))), VBox(children=(HTML(value='Forecasted: mean_slope_sto06'), SelectMultiple(index=(0, 1, 2), options=('swash', 'overwash', 'collision'), value=('swash', 'overwash', 'collision')))))))), HBox(children=(VBox(children=(HTML(value='<b>Filter by site_id:</b>'), HBox(children=(Dropdown(description='site_id:   ', index=943, options=('AVOCAn0001', 'AVOCAn0002', 'AVOCAn0003', 'AVOCAn0004', 'AVOCAn0005', 'AVOCAn0006', 'AVOCAn0007', 'AVOCAn0008', 'AVOCAn0009', 'AVOCAs0001', 'AVOCAs0002', 'AVOCAs0003', 'AVOCAs0004', 'AVOCAs0005', 'AVOCAs0006', 'AVOCAs0007', 'AVOCAs0008', 'BILG0001', 'BILG0002', 'BILG0003', 'BILG0004', 'BILG0005', 'BLUEYS0001', 'BLUEYS0002', 'BLUEYS0003', 'BLUEYS0004', 'BLUEYS0005', 'BLUEYS0006', 'BOAT0001', 'BOAT0002', 'BOAT0003', 'BOAT0004', 'BOAT0005', 'BOOM0001', 'BOOM0002', 'BOOM0003', 'BOOM0004', 'BOOM0005', 'BOOM0006', 'BOOM0007', 'BOOM0008', 'BOOM0009', 'BOOM0010', 'BOOM0011', 'BOOM0012', 'BOOM0013', 'BOOM0014', 'CATHIE0001', 'CATHIE0002', 'CATHIE0003', 'CATHIE0004', 'CATHIE0005', 'CATHIE0006', 'CATHIE0007', 'CATHIE0008', 'CATHIE0009', 'CATHIE0010', 'CATHIE0011', 'CATHIE0012', 'CATHIE0013', 'CATHIE0014', 'CATHIE0015', 'CATHIE0016', 'CATHIE0017', 'CATHIE0018', 'CATHIE0019', 'CATHIE0020', 'CATHIE0021', 'CATHIE0022', 'CATHIE0023', 'CATHIE0024', 'CATHIE0025', 'CATHIE0026', 'CATHIE0027', 'CATHIE0028', 'CATHIE0029', 'CRESn0001', 'CRESn0002', 'CRESn0003', 'CRESn0004', 'CRESn0005', 'CRESn0006', 'CRESn0007', 'CRESn0008', 'CRESn0009', 'CRESn0010', 'CRESn0011', 'CRESn0012', 'CRESn0013', 'CRESn0014', 'CRESn0015', 'CRESn0016', 'CRESn0017', 'CRESn0018', 'CRESn0019', 'CRESn0020', 'CRESn0021', 'CRESn0022', 'CRESn0023', 'CRESn0024', 'CRESn0025', 'CRESn0026', 'CRESn0027', 'CRESn0028', 'CRESn0029', 'CRESn0030', 'CRESn0031', 'CRESn0032', 'CRESn0033', 'CRESn0034', 'CRESn0035', 'CRESn0036', 'CRESn0037', 'CRESn0038', 'CRESn0039', 'CRESn0040', 'CRESn0041', 'CRESn0042', 'CRESn0043', 'CRESn0044', 'CRESn0045', 'CRESn0046', 'CRESn0047', 'CRESn0048', 'CRESn0049', 'CRESn0050', 'CRESn0051', 'CRESn0052', 'CRESn0053', 'CRESn0054', 'CRESn0055', 'CRESn0056', 'CRESn0057', 'CRESn0058', 'CRESn0059', 'CRESn0060', 'CRESn0061', 'CRESn0062', 'CRESn0063', 'CRESn0064', 'CRESn0065', 'CRESn0066', 'CRESn0067', 'CRESn0068', 'CRESn0069', 'CRESn0070', 'CRESn0071', 'CRESn0072', 'CRESn0073', 'CRESn0074', 'CRESn0075', 'CRESn0076', 'CRESn0077', 'CRESn0078', 'CRESn0079', 'CRESn0080', 'CRESn0081', 'CRESn0082', 'CRESn0083', 'CRESn0084', 'CRESn0085', 'CRESn0086', 'CRESn0087', 'CRESn0088', 'CRESn0089', 'CRESn0090', 'CRESn0091', 'CRESn0092', 'CRESn0093', 'CRESn0094', 'CRESn0095', 'CRESn0096', 'CRESn0097', 'CRESn0098', 'CRESn0099', 'CRESn0100', 'CRESn0101', 'CRESn0102', 'CRESn0103', 'CRESn0104', 'CRESn0105', 'CRESn0106', 'CRESn0107', 'CRESn0108', 'CRESn0109', 'CRESn0110', 'CRESn0111', 'CRESn0112', 'CRESn0113', 'CRESn0114', 'CRESn0115', 'CRESn0116', 'CRESn0117', 'CRESn0118', 'CRESn0119', 'CRESn0120', 'CRESn0121', 'CRESn0122', 'CRESn0123', 'CRESn0124', 'CRESn0125', 'CRESs0001', 'CRESs0002', 'CRESs0003', 'CRESs0004', 'CRESs0005', 'CRESs0006', 'CRESs0007', 'CRESs0008', 'CRESs0009', 'CRESs0010', 'CRESs0011', 'CRESs0012', 'CRESs0013', 'CRESs0014', 'DEEWHYn0001', 'DEEWHYn0002', 'DEEWHYn0003', 'DEEWHYn0004', 'DEEWHYn0005', 'DEEWHYn0006', 'DEEWHYn0007', 'DEEWHYn0008', 'DEEWHYn0009', 'DEEWHYn0010', 'DEEWHYn0011', 'DEEWHYn0012', 'DEEWHYs0001', 'DEEWHYs0002', 'DEEWHYs0003', 'DEEWHYs0004', 'DEEWHYs0005', 'DEEWHYs0006', 'DEEWHYs0007', 'DEEWHYs0008', 'DIAMONDn0001', 'DIAMONDn0002', 'DIAMONDn0003', 'DIAMONDn0004', 'DIAMONDn0005', 'DIAMONDn0006', 'DIAMONDn0007', 'DIAMONDn0008', 'DIAMONDn0009', 'DIAMONDn0010', 'DIAMONDn0011', 'DIAMONDn0012', 'DIAMONDn00
       "    'data': [{'line': {'color': 'rgb(51,160,44)', 'width': 2},\n",
       "              'name': 'Pre Storm Profile',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'dede4ee7-342d-41c2-b899-daa42347f70c',\n",
       "              'x': [0],\n",
       "              'y': [0]},\n",
       "             {'line': {'color': 'rgb(255,127,0)', 'width': 2},\n",
       "              'name': 'Post Storm Profile',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'd51ed30f-dc58-4334-a0d2-e0e76b9f3ff4',\n",
       "              'x': [0],\n",
       "              'y': [0]},\n",
       "             {'marker': {'color': 'rgba(255,255,255,0)', 'line': {'color': 'rgba(106,61,154, 1)', 'width': 2}, 'size': 10},\n",
       "              'mode': 'markers',\n",
       "              'name': 'Pre-storm dune crest',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'a09876d4-eebe-4a20-9f56-ddf10095c5de',\n",
       "              'x': [0],\n",
       "              'y': [0]},\n",
       "             {'marker': {'color': 'rgba(255,255,255,0)', 'line': {'color': 'rgba(202,178,214,1)', 'width': 2}, 'size': 10},\n",
       "              'mode': 'markers',\n",
       "              'name': 'Pre-storm dune toe',\n",
       "              'type': 'scatter',\n",
       "              'uid': '329c7dd8-2f5a-42e9-9813-81b6acb08419',\n",
       "              'x': [0],\n",
       "              'y': [0]},\n",
       "             {'line': {'color': 'rgb(44,127,184)', 'width': 4},\n",
       "              'mode': 'lines',\n",
       "              'name': 'Peak R_high: foreshore_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': '8cef75dd-a4eb-40de-8149-a6179a9c6ba6',\n",
       "              'x': [0],\n",
       "              'y': [0]},\n",
       "             {'line': {'color': 'rgb(127,205,187)', 'width': 4},\n",
       "              'mode': 'lines',\n",
       "              'name': 'Peak R_high: mean_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'c1691092-a100-4799-a807-13d8c2dc0679',\n",
       "              'x': [0],\n",
       "              'y': [0]}],\n",
       "    'layout': {'height': 300,\n",
       "               'legend': {'font': {'size': 10}},\n",
       "               'margin': {'b': 50, 'l': 50, 'r': 20, 't': 50},\n",
       "               'title': 'Bed Profiles',\n",
       "               'xaxis': {'autorange': True,\n",
       "                         'range': [0, 200],\n",
       "                         'showgrid': True,\n",
       "                         'showline': True,\n",
       "                         'title': 'x (m)',\n",
       "                         'zeroline': True},\n",
       "               'yaxis': {'autorange': False,\n",
       "                         'range': [-1, 20],\n",
       "                         'showgrid': True,\n",
       "                         'showline': True,\n",
       "                         'title': 'z (m)',\n",
       "                         'zeroline': True}}\n",
       "}), FigureWidget({\n",
       "    'data': [{'lat': array([-33.45779575, -33.4584035 , -33.45911131, ..., -33.44589876,\n",
       "                            -33.44665247, -33.44732817]),\n",
       "              'lon': array([151.44105489, 151.44024676, 151.43957804, ..., 151.44420973,\n",
       "                            151.44479881, 151.44550951]),\n",
       "              'marker': {'size': 10},\n",
       "              'mode': 'markers',\n",
       "              'text': array(['AVOCAn0001', 'AVOCAn0002', 'AVOCAn0003', ..., 'WAMBE0025', 'WAMBE0026',\n",
       "                             'WAMBE0027'], dtype='<U12'),\n",
       "              'type': 'scattermapbox',\n",
       "              'uid': 'c2b2c569-23f0-4c3c-893b-2d62701659bb'},\n",
       "             {'lat': [0],\n",
       "              'lon': [0],\n",
       "              'marker': {'color': 'rgb(255, 0, 0)', 'opacity': 0.5, 'size': 20},\n",
       "              'mode': 'markers',\n",
       "              'text': array(['AVOCAn0001', 'AVOCAn0002', 'AVOCAn0003', ..., 'WAMBE0025', 'WAMBE0026',\n",
       "                             'WAMBE0027'], dtype='<U12'),\n",
       "              'type': 'scattermapbox',\n",
       "              'uid': 'b79aaf58-78ee-4ef4-911d-8ca70bc96b03'}],\n",
       "    'layout': {'autosize': True,\n",
       "               'height': 300,\n",
       "               'hovermode': 'closest',\n",
       "               'mapbox': {'accesstoken': ('pk.eyJ1IjoiY2hyaXNsZWFtYW4iLCJ' ... 'Hp5bCJ9.U2dwFg2c7RFjUNSayERUiw'),\n",
       "                          'bearing': 0,\n",
       "                          'center': {'lat': -33.7, 'lon': 151.3},\n",
       "                          'pitch': 0,\n",
       "                          'style': 'satellite-streets',\n",
       "                          'zoom': 12},\n",
       "               'margin': {'b': 50, 'l': 20, 'r': 20, 't': 50},\n",
       "               'showlegend': False}\n",
       "}))), FigureWidget({\n",
       "    'data': [{'name': 'Hs0',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'db238b0e-ee15-49f5-8766-b67c88d9e605',\n",
       "              'x': [0, 1],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 1],\n",
       "              'yaxis': 'y3'},\n",
       "             {'name': 'Tp',\n",
       "              'type': 'scatter',\n",
       "              'uid': '7ecb1b84-1cf0-4f3c-95e4-161ca9d6b53e',\n",
       "              'x': [0, 1],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 1],\n",
       "              'yaxis': 'y4'},\n",
       "             {'line': {'color': 'rgb(214, 117, 14)', 'dash': 'dot', 'width': 2},\n",
       "              'mode': 'lines',\n",
       "              'name': 'Dune Crest',\n",
       "              'type': 'scatter',\n",
       "              'uid': '8506c10f-7f45-46b7-9a19-a503b78fec8a',\n",
       "              'x': [0, 3],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 3],\n",
       "              'yaxis': 'y'},\n",
       "             {'line': {'color': 'rgb(142, 77, 8)', 'dash': 'dash', 'width': 2},\n",
       "              'mode': 'lines',\n",
       "              'name': 'Dune Toe',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'bcbd9782-3c57-4267-baa1-3601fabaeee4',\n",
       "              'x': [0, 3],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 3],\n",
       "              'yaxis': 'y'},\n",
       "             {'line': {'color': 'rgb(8,51,137)', 'dash': 'dot', 'width': 2},\n",
       "              'name': 'Tide+Surge WL',\n",
       "              'type': 'scatter',\n",
       "              'uid': '3d44a507-891a-491d-9057-706c1b7388f3',\n",
       "              'x': [0, 3],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 3],\n",
       "              'yaxis': 'y'},\n",
       "             {'line': {'color': 'rgb(44,127,184)', 'width': 3},\n",
       "              'name': 'R_high: foreshore_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': '45ccf4de-d026-470d-a4a4-1a8dd830440a',\n",
       "              'x': [0],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0],\n",
       "              'yaxis': 'y'},\n",
       "             {'line': {'color': 'rgb(127,205,187)', 'width': 3},\n",
       "              'name': 'R_high: mean_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': '14ef23fe-a786-41de-a833-3df985b64c50',\n",
       "              'x': [0],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0],\n",
       "              'yaxis': 'y'},\n",
       "             {'line': {'color': 'rgb(44,127,184)', 'width': 3},\n",
       "              'name': 'Beta: foreshore_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': 'd2cd0cc9-b9f6-432c-a050-016a04f42957',\n",
       "              'x': [0, 1],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 1],\n",
       "              'yaxis': 'y2'},\n",
       "             {'line': {'color': 'rgb(127,205,187)', 'width': 3},\n",
       "              'name': 'Beta: mean_slope_sto06',\n",
       "              'type': 'scatter',\n",
       "              'uid': '6d14a303-7630-4b0f-ba65-d3949a30fc7a',\n",
       "              'x': [0, 1],\n",
       "              'xaxis': 'x',\n",
       "              'y': [0, 1],\n",
       "              'yaxis': 'y2'}],\n",
       "    'layout': {'height': 400,\n",
       "               'legend': {'font': {'size': 10}},\n",
       "               'margin': {'b': 100, 'l': 50, 'r': 20, 't': 20},\n",
       "               'xaxis': {'anchor': 'y3', 'domain': [0.0, 1.0], 'title': 'datetime'},\n",
       "               'yaxis': {'anchor': 'free', 'domain': [0.7333333333333333, 1.0], 'position': 0.0, 'title': 'z (mAHD)'},\n",
       "               'yaxis2': {'anchor': 'free',\n",
       "                          'domain': [0.36666666666666664, 0.6333333333333333],\n",
       "                          'position': 0.0,\n",
       "                          'title': 'beta (-)'},\n",
       "               'yaxis3': {'anchor': 'x', 'domain': [0.0, 0.26666666666666666], 'title': 'Hs0 (m)'},\n",
       "               'yaxis4': {'overlaying': 'y3', 'side': 'right', 'title': 'Tp (s)'}}\n",
       "}), HBox(children=(VBox(children=(HTML(value='observed'), Output())), VBox(children=(HTML(value='foreshore_slope_sto06'), Output())), VBox(children=(HTML(value='mean_slope_sto06'), Output()))))))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create widgets for filtering by observed and forecasted impacts\n",
    "filter_title = widgets.HTML(\n",
    "    value=\"<b>Filter by observed and predicted impacts:</b>\", )\n",
    "\n",
    "titles = ['Observed Impacts']\n",
    "selectboxes = [\n",
    "    widgets.SelectMultiple(\n",
    "        options=impacts['observed'].storm_regime.dropna().unique().tolist(),\n",
    "        value=impacts['observed'].storm_regime.dropna().unique().tolist(),\n",
    "        disabled=False)\n",
    "]\n",
    "\n",
    "# Iterate through each of our forecasted impacts\n",
    "for forecast in impacts['forecasted']:\n",
    "    selectboxes.append(\n",
    "        widgets.SelectMultiple(\n",
    "            options=impacts['forecasted'][\n",
    "                forecast].storm_regime.dropna().unique().tolist(),\n",
    "            value=impacts['forecasted'][forecast].storm_regime.dropna()\n",
    "            .unique().tolist(),\n",
    "            disabled=False))\n",
    "    titles.append('Forecasted: {}'.format(forecast))\n",
    "\n",
    "titles = [widgets.HTML(value=title) for title in titles]\n",
    "\n",
    "children = widgets.HBox(children=[\n",
    "    widgets.VBox(children=[title, box])\n",
    "    for title, box in zip(titles, selectboxes)\n",
    "])\n",
    "filter_container = widgets.VBox(children=[filter_title, children])\n",
    "\n",
    "# Create widgets for selecting site_id\n",
    "site_id_title = widgets.HTML(value=\"<b>Filter by site_id:</b>\", )\n",
    "\n",
    "site_id_select = widgets.Dropdown(\n",
    "    description='site_id:   ',\n",
    "    value='NARRA0001',\n",
    "    options=df_profiles.index.get_level_values('site_id').unique()\n",
    "    .sort_values().tolist())\n",
    "\n",
    "site_id_impacts = widgets.HTML(value=\"\", )\n",
    "\n",
    "site_id_container = widgets.HBox(children=[\n",
    "    widgets.VBox(\n",
    "        children=[site_id_title,\n",
    "                  widgets.HBox(children=[site_id_select])]), site_id_impacts\n",
    "])\n",
    "\n",
    "# Build colors for each of our forecasts\n",
    "colors = list(\n",
    "    reversed(cl.scales[str(max(len(impacts['forecasted']),\n",
    "                               3))]['seq']['YlGnBu']))\n",
    "\n",
    "# Add panel for pre/post storm profiles\n",
    "trace1 = go.Scatter(\n",
    "    x=[0],\n",
    "    y=[0],\n",
    "    name='Pre Storm Profile',\n",
    "    line=dict(color=('rgb(51,160,44)'), width=2))\n",
    "trace2 = go.Scatter(\n",
    "    x=[0],\n",
    "    y=[0],\n",
    "    name='Post Storm Profile',\n",
    "    line=dict(color=('rgb(255,127,0)'), width=2))\n",
    "trace3 = go.Scatter(\n",
    "    x=[0],\n",
    "    y=[0],\n",
    "    name='Pre-storm dune crest',\n",
    "    mode='markers',\n",
    "    marker=dict(\n",
    "        color='rgba(255,255,255,0)',\n",
    "        size=10,\n",
    "        line=dict(color='rgba(106,61,154, 1)', width=2)),\n",
    ")\n",
    "trace4 = go.Scatter(\n",
    "    x=[0],\n",
    "    y=[0],\n",
    "    name='Pre-storm dune toe',\n",
    "    mode='markers',\n",
    "    marker=dict(\n",
    "        color='rgba(255,255,255,0)',\n",
    "        size=10,\n",
    "        line=dict(color='rgba(202,178,214,1)', width=2)),\n",
    ")\n",
    "\n",
    "forecast_traces = []\n",
    "for forecast, color in zip(impacts['forecasted'], colors):\n",
    "    forecast_traces.append(\n",
    "        go.Scatter(\n",
    "            x=[0],\n",
    "            y=[0],\n",
    "            name='Peak R_high: {}'.format(forecast),\n",
    "            mode='lines',\n",
    "            line=dict(\n",
    "                color=color,\n",
    "                width=4,\n",
    "            )))\n",
    "\n",
    "layout = go.Layout(\n",
    "    title='Bed Profiles',\n",
    "    height=300,\n",
    "    legend=dict(font={'size': 10}),\n",
    "    margin=dict(t=50, b=50, l=50, r=20),\n",
    "    xaxis=dict(\n",
    "        title='x (m)',\n",
    "        autorange=True,\n",
    "        showgrid=True,\n",
    "        zeroline=True,\n",
    "        showline=True,\n",
    "        range=[0, 200]),\n",
    "    yaxis=dict(\n",
    "        title='z (m)',\n",
    "        autorange=False,\n",
    "        showgrid=True,\n",
    "        zeroline=True,\n",
    "        showline=True,\n",
    "        range=[-1, 20]))\n",
    "\n",
    "g_profiles = go.FigureWidget(\n",
    "    data=[trace1, trace2, trace3, trace4] + forecast_traces, layout=layout)\n",
    "\n",
    "# Add panel for google maps\n",
    "mapbox_access_token = 'pk.eyJ1IjoiY2hyaXNsZWFtYW4iLCJhIjoiY2pvNTY1MzZpMDc2OTN2bmw5MGsycHp5bCJ9.U2dwFg2c7RFjUNSayERUiw'\n",
    "\n",
    "data = [\n",
    "    go.Scattermapbox(\n",
    "        lat=df_sites['lat'],\n",
    "        lon=df_sites['lon'],\n",
    "        mode='markers',\n",
    "        marker=dict(size=10),\n",
    "        text=df_sites.index.get_level_values('site_id'),\n",
    "    ),\n",
    "    go.Scattermapbox(\n",
    "        lat=[0],\n",
    "        lon=[0],\n",
    "        mode='markers',\n",
    "        marker=dict(\n",
    "            size=20,\n",
    "            color='rgb(255, 0, 0)',\n",
    "            opacity=0.5,\n",
    "        ),\n",
    "        text=df_sites.index.get_level_values('site_id'),\n",
    "    ),\n",
    "]\n",
    "\n",
    "layout = go.Layout(\n",
    "    autosize=True,\n",
    "    height=300,\n",
    "    hovermode='closest',\n",
    "    showlegend=False,\n",
    "    margin=dict(t=50, b=50, l=20, r=20),\n",
    "    mapbox=dict(\n",
    "        accesstoken=mapbox_access_token,\n",
    "        bearing=0,\n",
    "        center=dict(lat=-33.7, lon=151.3),\n",
    "        pitch=0,\n",
    "        zoom=12,\n",
    "        style='satellite-streets'),\n",
    ")\n",
    "\n",
    "fig = dict(data=data, layout=layout)\n",
    "g_map = go.FigureWidget(data=data, layout=layout)\n",
    "\n",
    "subplot = tls.make_subplots(3, 1, print_grid=False, shared_xaxes=True)\n",
    "g_timeseries = go.FigureWidget(subplot)\n",
    "\n",
    "# Add trace for Hs0\n",
    "g_timeseries.add_trace(\n",
    "    go.Scatter(\n",
    "        x=[0, 1],\n",
    "        y=[0, 1],\n",
    "        name='Hs0',\n",
    "    ),\n",
    "    row=3,\n",
    "    col=1,\n",
    ")\n",
    "\n",
    "# Add trace for Tp\n",
    "g_timeseries.add_trace(\n",
    "    go.Scatter(\n",
    "        x=[0, 1],\n",
    "        y=[0, 1],\n",
    "        name='Tp',\n",
    "    ),\n",
    "    row=3,\n",
    "    col=1,\n",
    ")\n",
    "\n",
    "# Add water levels\n",
    "g_timeseries.add_trace(\n",
    "    go.Scatter(\n",
    "        x=[0, 3],\n",
    "        y=[0, 3],\n",
    "        name='Dune Crest',\n",
    "        mode='lines',\n",
    "        line=dict(color=('rgb(214, 117, 14)'), width=2, dash='dot')),\n",
    "    row=1,\n",
    "    col=1)\n",
    "\n",
    "g_timeseries.add_trace(\n",
    "    go.Scatter(\n",
    "        x=[0, 3],\n",
    "        y=[0, 3],\n",
    "        name='Dune Toe',\n",
    "        mode='lines',\n",
    "        line=dict(color=('rgb(142, 77, 8)'), width=2, dash='dash')),\n",
    "    row=1,\n",
    "    col=1)\n",
    "\n",
    "g_timeseries.add_trace(\n",
    "    go.Scatter(\n",
    "        x=[0, 3],\n",
    "        y=[0, 3],\n",
    "        name='Tide+Surge WL',\n",
    "        line=dict(color=('rgb(8,51,137)'), width=2, dash='dot')),\n",
    "    row=1,\n",
    "    col=1)\n",
    "\n",
    "for forecast, color in zip(twls['forecasted'], colors):\n",
    "    g_timeseries.add_trace(\n",
    "        go.Scatter(\n",
    "            x=[0],\n",
    "            y=[0],\n",
    "            name='R_high: {}'.format(forecast),\n",
    "            line=dict(color=color, width=3)),\n",
    "        row=1,\n",
    "        col=1)\n",
    "\n",
    "# Add trace for each forecasted beta term\n",
    "for forecast, color in zip(impacts['forecasted'], colors):\n",
    "    g_timeseries.add_trace(\n",
    "        go.Scatter(\n",
    "            x=[0, 1],\n",
    "            y=[0, 1],\n",
    "            name='Beta: {}'.format(forecast),\n",
    "            line=dict(color=color, width=3)),\n",
    "        row=2,\n",
    "        col=1,\n",
    "    )\n",
    "\n",
    "# Create axis for Tp on same plot as Hs\n",
    "g_timeseries['layout']['yaxis4'] = {'overlaying': 'y3', 'side': 'right'}\n",
    "g_timeseries.data[1]['yaxis'] = 'y4'\n",
    "\n",
    "# Add labels to each axis\n",
    "g_timeseries.layout['xaxis']['title'] = 'datetime'\n",
    "g_timeseries.layout['yaxis1']['title'] = 'z (mAHD)'\n",
    "g_timeseries.layout['yaxis2']['title'] = 'beta (-)'\n",
    "g_timeseries.layout['yaxis3']['title'] = 'Hs0 (m)'\n",
    "g_timeseries.layout['yaxis4']['title'] = 'Tp (s)'\n",
    "\n",
    "# Update figure size\n",
    "g_timeseries['layout'].update(height=400, legend=dict(font={'size': 10}))\n",
    "g_timeseries['layout'].update(margin=dict(t=20, l=50, r=20, b=100))\n",
    "\n",
    "# Add panel for some tables\n",
    "titles = ['observed'] + [forecast for forecast in impacts['forecasted']]\n",
    "titles = [widgets.HTML(value=\"{}\".format(title)) for title in titles]\n",
    "\n",
    "\n",
    "def get_observed_impacts_table(site_id):\n",
    "    display(impacts['observed'].query(\"site_id=='{}'\".format(site_id)).T)\n",
    "\n",
    "\n",
    "def get_forecasted_impacts_table(site_id, forecast):\n",
    "    display(impacts['forecasted'][forecast].query(\n",
    "        \"site_id=='{}'\".format(site_id)).T)\n",
    "\n",
    "\n",
    "impacts_table_observed = widgets.interactive_output(\n",
    "    get_observed_impacts_table, {'site_id': site_id_select})\n",
    "forecasted_impacts_tables = []\n",
    "for forecast, title in zip(impacts['forecasted'], titles[1:]):\n",
    "    forecasted_impacts_tables.append(\n",
    "        widgets.interactive_output(get_forecasted_impacts_table, {\n",
    "            'site_id': site_id_select,\n",
    "            'forecast': title\n",
    "        }))\n",
    "\n",
    "tables = [impacts_table_observed] + forecasted_impacts_tables\n",
    "\n",
    "title_tables = [\n",
    "    widgets.VBox(children=[title, table])\n",
    "    for title, table in zip(titles, tables)\n",
    "]\n",
    "\n",
    "tables_container = widgets.HBox(children=[*title_tables])\n",
    "\n",
    "\n",
    "def update_profile(change):\n",
    "\n",
    "    site_id = site_id_select.value\n",
    "\n",
    "    if site_id is None:\n",
    "        return\n",
    "\n",
    "    site_profile = df_profiles.query('site_id == \"{}\"'.format(site_id))\n",
    "    prestorm_profile = site_profile.query('profile_type == \"prestorm\"')\n",
    "    poststorm_profile = site_profile.query('profile_type == \"poststorm\"')\n",
    "\n",
    "    poststorm_x = poststorm_profile.index.get_level_values('x').tolist()\n",
    "    poststorm_z = poststorm_profile.z.tolist()\n",
    "\n",
    "    prestorm_x = prestorm_profile.index.get_level_values('x').tolist()\n",
    "    prestorm_z = prestorm_profile.z.tolist()\n",
    "\n",
    "    site_features = df_profile_features.query(\n",
    "        'site_id == \"{}\"'.format(site_id))\n",
    "    dune_crest_x = site_features.dune_crest_x\n",
    "    dune_crest_z = site_features.dune_crest_z\n",
    "    dune_toe_x = site_features.dune_toe_x\n",
    "    dune_toe_z = site_features.dune_toe_z\n",
    "\n",
    "    # Update beach profile section plots\n",
    "    with g_profiles.batch_update():\n",
    "        g_profiles.data[0].x = prestorm_x\n",
    "        g_profiles.data[0].y = prestorm_z\n",
    "        g_profiles.data[1].x = poststorm_x\n",
    "        g_profiles.data[1].y = poststorm_z\n",
    "        g_profiles.data[2].x = dune_crest_x\n",
    "        g_profiles.data[2].y = dune_crest_z\n",
    "        g_profiles.data[3].x = dune_toe_x\n",
    "        g_profiles.data[3].y = dune_toe_z\n",
    "\n",
    "        for n, forecast in enumerate(impacts['forecasted']):\n",
    "            R_high = max(impacts['forecasted'][forecast].query(\n",
    "                \"site_id=='{}'\".format(site_id)).R_high)\n",
    "            g_profiles.data[4 + n].x = [200, 400]\n",
    "            g_profiles.data[4 + n].y = [R_high, R_high]\n",
    "\n",
    "    # Relocate plan of satellite imagery\n",
    "    site_coords = df_sites.query('site_id == \"{}\"'.format(site_id))\n",
    "    with g_map.batch_update():\n",
    "        g_map.layout.mapbox['center'] = {\n",
    "            'lat': site_coords['lat'].values[0],\n",
    "            'lon': site_coords['lon'].values[0]\n",
    "        }\n",
    "        g_map.layout.mapbox['zoom'] = 15\n",
    "        g_map.data[1].lat = [site_coords['lat'].values[0]]\n",
    "        g_map.data[1].lon = [site_coords['lon'].values[0]]\n",
    "        g_map.data[1].text = site_coords['lon'].index.get_level_values(\n",
    "            'site_id').tolist()\n",
    "\n",
    "    # Update time series plots\n",
    "    df_waves_site = df_waves.query(\"site_id=='{}'\".format(site_id))\n",
    "    times = df_waves_site.index.get_level_values('datetime').tolist()\n",
    "    Hs0s = df_waves_site.Hs0.tolist()\n",
    "    Tps = df_waves_site.Tp.tolist()\n",
    "\n",
    "    df_tide_site = df_tides.query(\"site_id=='{}'\".format(site_id))\n",
    "    mask = (df_tide_site.index.get_level_values('datetime') >= min(times)) & (\n",
    "        df_tide_site.index.get_level_values('datetime') <= max(times))\n",
    "    df_tide_site = df_tide_site.loc[mask]\n",
    "\n",
    "    with g_timeseries.batch_update():\n",
    "        g_timeseries.data[0].x = times\n",
    "        g_timeseries.data[0].y = Hs0s\n",
    "        g_timeseries.data[1].x = times\n",
    "        g_timeseries.data[1].y = Tps\n",
    "\n",
    "        # Update beta values\n",
    "        idx_betas = [\n",
    "            n for n, x in enumerate(g_timeseries.data) if 'Beta' in x.name\n",
    "        ]\n",
    "        for i, forecast in zip(idx_betas, twls['forecasted']):\n",
    "            df_twl = twls['forecasted'][forecast].query(\n",
    "                \"site_id=='{}'\".format(site_id))\n",
    "            times = df_twl.index.get_level_values('datetime').tolist()\n",
    "            beta = df_twl.beta.tolist()\n",
    "            g_timeseries.data[i].x = times\n",
    "            g_timeseries.data[i].y = beta\n",
    "\n",
    "        g_timeseries.data[2].x = [min(times), max(times)]\n",
    "        g_timeseries.data[3].x = [min(times), max(times)]\n",
    "        g_timeseries.data[4].x = df_tide_site.index.get_level_values(\n",
    "            'datetime')\n",
    "        g_timeseries.data[2].y = dune_crest_z.tolist()[\n",
    "            0], dune_crest_z.tolist()[0],\n",
    "        g_timeseries.data[3].y = dune_toe_z.tolist()[0], dune_toe_z.tolist()[\n",
    "            0],\n",
    "        g_timeseries.data[4].y = df_tide_site.tide.tolist()\n",
    "\n",
    "        # Update rhigh values\n",
    "        idx_betas = [\n",
    "            n for n, x in enumerate(g_timeseries.data) if 'R_high' in x.name\n",
    "        ]\n",
    "        for i, forecast in zip(idx_betas, twls['forecasted']):\n",
    "            df_twl = twls['forecasted'][forecast].query(\n",
    "                \"site_id=='{}'\".format(site_id))\n",
    "            times = df_twl.index.get_level_values('datetime').tolist()\n",
    "            R_high = df_twl.R_high.tolist()\n",
    "            g_timeseries.data[i].x = times\n",
    "            g_timeseries.data[i].y = R_high\n",
    "\n",
    "    # Update site id impacts\n",
    "    observed_regime = impacts['observed'].query(\n",
    "        \"site_id=='{}'\".format(site_id)).storm_regime.values[0]\n",
    "    site_id_impacts.value = \"Observed: <b>{}</b><br>\".format(\n",
    "        observed_regime)\n",
    "\n",
    "    for forecast in impacts['forecasted']:\n",
    "        regime = impacts['forecasted'][forecast].query(\n",
    "            \"site_id=='{}'\".format(site_id)).storm_regime.values[0]\n",
    "        site_id_impacts.value += '{}: <b>{}</b><br>'.format(\n",
    "            forecast, regime)\n",
    "\n",
    "\n",
    "site_id_select.observe(update_profile, names=\"value\")\n",
    "\n",
    "\n",
    "def update_filter(change):\n",
    "\n",
    "    # Iterate through each box, only keeping site_ids which are not filtered out by each box\n",
    "    valid_site_ids = impacts['observed'].index.tolist()\n",
    "    dfs = [impacts['observed']\n",
    "           ] + [impacts['forecasted'][key] for key in impacts['forecasted']]\n",
    "\n",
    "    for box, df in zip(selectboxes, dfs):\n",
    "        valid_site_ids = list(\n",
    "            set(valid_site_ids).intersection(\n",
    "                set(df[df.storm_regime.isin(box.value)].index.tolist())))\n",
    "    site_id_select.options = sorted(valid_site_ids)\n",
    "\n",
    "    # TODO Update options in selectboxes with number of observations?\n",
    "\n",
    "\n",
    "# Update the filter if any of the boxes changes\n",
    "for box in selectboxes:\n",
    "    box.observe(update_filter, names=\"value\")\n",
    "\n",
    "# Display our widgets!\n",
    "widgets.VBox([\n",
    "    filter_container, site_id_container,\n",
    "    widgets.HBox([g_profiles, g_map]), g_timeseries, tables_container\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-22T22:52:36.039701Z",
     "start_time": "2018-11-22T22:52:36.035189Z"
    },
    "scrolled": true
   },
   "source": [
    "This visualization looks at how well the storm impact predictions performed. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:35.111533Z",
     "start_time": "2018-11-27T03:27:34.422864Z"
    },
    "code_folding": [],
    "hide_input": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "917e00e9b4814276998243493e5b87d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "VBox(children=(VBox(children=(HTML(value='<b>Filter by beach:</b>'), SelectMultiple(index=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46), options=('AVOCAn', 'AVOCAs', 'BILG', 'BLUEYS', 'BOAT', 'BOOM', 'CATHIE', 'CRESn', 'CRESs', 'DEEWHYn', 'DEEWHYs', 'DIAMONDn', 'DIAMONDs', 'DUNBn', 'DUNBs', 'ELIZA', 'ENTRA', 'FOST', 'GRANTSn', 'GRANTSs', 'HARGn', 'HARGs', 'HARR', 'LHOUSE', 'LHOUSEn', 'LHOUSEs', 'MACM', 'MANNING', 'MONA', 'NAMB', 'NARRA', 'NINEMn', 'NINEMs', 'NSHORE_n', 'NSHORE_s', 'OLDBAR', 'ONEMILE', 'PEARLn', 'PEARLs', 'SCOT', 'STOCNn', 'STOCNs', 'STOCS', 'STUART', 'SWRO', 'TREACH', 'WAMBE'), value=('AVOCAn', 'AVOCAs', 'BILG', 'BLUEYS', 'BOAT', 'BOOM', 'CATHIE', 'CRESn', 'CRESs', 'DEEWHYn', 'DEEWHYs', 'DIAMONDn', 'DIAMONDs', 'DUNBn', 'DUNBs', 'ELIZA', 'ENTRA', 'FOST', 'GRANTSn', 'GRANTSs', 'HARGn', 'HARGs', 'HARR', 'LHOUSE', 'LHOUSEn', 'LHOUSEs', 'MACM', 'MANNING', 'MONA', 'NAMB', 'NARRA', 'NINEMn', 'NINEMs', 'NSHORE_n', 'NSHORE_s', 'OLDBAR', 'ONEMILE', 'PEARLn', 'PEARLs', 'SCOT', 'STOCNn', 'STOCNs', 'STOCS', 'STUART', 'SWRO', 'TREACH', 'WAMBE')))), VBox(children=(VBox(children=(HTML(value='<b>foreshore_slope_sto06</b>'), FigureWidget({\n",
       "    'data': [{'colorscale': [[0.0, 'rgb(165, 0, 38)'], [0.003937007874015748,\n",
       "                             'rgb(166, 1, 38)'], [0.007874015748031496, 'rgb(168,\n",
       "                             3, 38)'], ..., [0.9921259842519685, 'rgb(2, 107,\n",
       "                             56)'], [0.9960629921259843, 'rgb(1, 105, 55)'], [1.0,\n",
       "                             'rgb(0, 104, 55)']],\n",
       "              'reversescale': False,\n",
       "              'showscale': False,\n",
       "              'type': 'heatmap',\n",
       "              'uid': 'aaeea549-ca69-4311-9d7b-7d048578061d',\n",
       "              'x': [swash, collision, overwash, inundation],\n",
       "              'y': [inundation, overwash, collision, swash],\n",
       "              'z': [[0.1, 0.3, 0.5, 2], [1.0, 0.8, 0.6, 1], [1.4, 0.28, 1.6,\n",
       "                    0.21], [0.6, 0.4, 0.2, 3]]}],\n",
       "    'layout': {'annotations': [{'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.1',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.3',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.5',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '2',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.0',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.8',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.6',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.4',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.28',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.6',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.21',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.6',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.4',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.2',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '3',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'}],\n",
       "               'height': 200,\n",
       "               'margin': {'b': 40, 'l': 100, 'pad': 0, 'r': 100, 't': 40},\n",
       "               'xaxis': {'dtick': 1, 'gridcolor': 'rgb(0, 0, 0)', 'side': 'top', 'ticks': '', 'title': 'Predicted'},\n",
       "               'yaxis': {'dtick': 1, 'ticks': '', 'ticksuffix': '  ', 'title': 'Observed'}}\n",
       "}))), VBox(children=(HTML(value='<b>mean_slope_sto06</b>'), FigureWidget({\n",
       "    'data': [{'colorscale': [[0.0, 'rgb(165, 0, 38)'], [0.003937007874015748,\n",
       "                             'rgb(166, 1, 38)'], [0.007874015748031496, 'rgb(168,\n",
       "                             3, 38)'], ..., [0.9921259842519685, 'rgb(2, 107,\n",
       "                             56)'], [0.9960629921259843, 'rgb(1, 105, 55)'], [1.0,\n",
       "                             'rgb(0, 104, 55)']],\n",
       "              'reversescale': False,\n",
       "              'showscale': False,\n",
       "              'type': 'heatmap',\n",
       "              'uid': 'c36240cc-9418-4afa-8b08-af02844dcad7',\n",
       "              'x': [swash, collision, overwash, inundation],\n",
       "              'y': [inundation, overwash, collision, swash],\n",
       "              'z': [[0.1, 0.3, 0.5, 2], [1.0, 0.8, 0.6, 1], [1.4, 0.28, 1.6,\n",
       "                    0.21], [0.6, 0.4, 0.2, 3]]}],\n",
       "    'layout': {'annotations': [{'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.1',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.3',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.5',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '2',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'inundation',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.0',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.8',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.6',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'overwash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.4',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.28',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '1.6',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.21',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'collision',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.6',\n",
       "                                'x': 'swash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.4',\n",
       "                                'x': 'collision',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '0.2',\n",
       "                                'x': 'overwash',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'},\n",
       "                               {'font': {'color': '#FFFFFF'},\n",
       "                                'showarrow': False,\n",
       "                                'text': '3',\n",
       "                                'x': 'inundation',\n",
       "                                'xref': 'x',\n",
       "                                'y': 'swash',\n",
       "                                'yref': 'y'}],\n",
       "               'height': 200,\n",
       "               'margin': {'b': 40, 'l': 100, 'pad': 0, 'r': 100, 't': 40},\n",
       "               'xaxis': {'dtick': 1, 'gridcolor': 'rgb(0, 0, 0)', 'side': 'top', 'ticks': '', 'title': 'Predicted'},\n",
       "               'yaxis': {'dtick': 1, 'ticks': '', 'ticksuffix': '  ', 'title': 'Observed'}}\n",
       "})))))))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create colorscale\n",
    "rdylgr_cmap = matplotlib.cm.get_cmap('RdYlGn')\n",
    "\n",
    "norm = matplotlib.colors.Normalize(vmin=0, vmax=255)\n",
    "\n",
    "def matplotlib_to_plotly(cmap, pl_entries):\n",
    "    h = 1.0/(pl_entries-1)\n",
    "    pl_colorscale = []\n",
    "\n",
    "    for k in range(pl_entries):\n",
    "        C = list(map(np.uint8, np.array(cmap(k*h)[:3])*255))\n",
    "        pl_colorscale.append([k*h, 'rgb'+str((C[0], C[1], C[2]))])\n",
    "\n",
    "    return pl_colorscale\n",
    "\n",
    "rdylgr = matplotlib_to_plotly(rdylgr_cmap, 255)\n",
    "\n",
    "\n",
    "\n",
    "# Create widget for list of beaches.\n",
    "beaches = df_sites.beach.unique().tolist()\n",
    "\n",
    "beach_title = widgets.HTML(value=\"<b>Filter by beach:</b>\", )\n",
    "\n",
    "beach_select = widgets.SelectMultiple(\n",
    "    options=beaches, value=beaches, disabled=False)\n",
    "\n",
    "beach_container = widgets.VBox([beach_title, beach_select])\n",
    "\n",
    "# Create confusion matrix for each forecasted impact data set\n",
    "heatmaps = []\n",
    "for forecast in impacts['forecasted']:\n",
    "\n",
    "    z = [[.1, .3, .5, 2], [1.0, .8, .6, 1], [1.4, .28, 1.6, .21],\n",
    "         [.6, .4, .2, 3]]\n",
    "\n",
    "    x = ['swash', 'collision', 'overwash', 'inundation']\n",
    "    y = list(reversed(x))\n",
    "\n",
    "    z_text = z\n",
    "\n",
    "    fig = ff.create_annotated_heatmap(z, x=x, y=y, annotation_text=z_text, colorscale=rdylgr)\n",
    "    heatmap = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    heatmap.layout.update(\n",
    "        height=200, margin=go.layout.Margin(l=100, r=100, b=40, t=40, pad=0))\n",
    "    heatmap.layout.xaxis.update(title='Predicted')\n",
    "    heatmap.layout.yaxis.update(title='Observed')\n",
    "    heatmap_title = widgets.HTML(value=\"<b>{}</b>\".format(forecast) )\n",
    "    heatmaps.append(widgets.VBox([heatmap_title, heatmap]))\n",
    "\n",
    "    \n",
    "def update_heatmaps(change):\n",
    "    \n",
    "    for forecast, heatmap in zip(impacts['forecasted'], heatmaps):\n",
    "        selected_site_ids = df_sites[df_sites.beach.isin(beach_select.value)].index.tolist()\n",
    "\n",
    "        df_ob = impacts['observed']\n",
    "        df_fo = impacts['forecasted'][forecast]\n",
    "\n",
    "        observed_regimes = df_ob[df_ob.index.isin(selected_site_ids)].storm_regime.dropna().rename(\"observed_regime\")\n",
    "        forecasted_regimes = df_fo[df_fo.index.isin(selected_site_ids)].storm_regime.dropna().rename(\"forecasted_regime\")\n",
    "\n",
    "        if any([observed_regimes.empty, forecasted_regimes.empty]):\n",
    "            return\n",
    "        \n",
    "        df_compare = pd.concat([observed_regimes, forecasted_regimes], axis='columns', names=['a','b'], sort=True)\n",
    "        df_compare.dropna(axis='index',inplace=True)\n",
    "\n",
    "        z = confusion_matrix(df_compare.observed_regime.tolist(), df_compare.forecasted_regime.tolist(), labels = ['swash','collision','overwash','inundation'])\n",
    "        z = np.flip(z,axis=0)\n",
    "        z_list = list(reversed(z.tolist()))\n",
    "        \n",
    "        # Make incorrect values negative, so they get assigned a different color.\n",
    "        # Better for visualization\n",
    "        z_neg_incorrect = np.flip(np.identity(4),axis=0)\n",
    "        z_neg_incorrect[z_neg_incorrect==0]= -1\n",
    "        z_neg_incorrect = (z * z_neg_incorrect).tolist()\n",
    "        \n",
    "        fig = ff.create_annotated_heatmap(z_neg_incorrect, x=x, y=y, annotation_text=z)\n",
    "        heatmap.children[1].data[0].z = z_neg_incorrect\n",
    "        heatmap.children[1].layout.annotations = fig.layout.annotations\n",
    "\n",
    "# Hook changes to beach filter to update confusion heatmaps\n",
    "beach_select.observe(update_heatmaps, names=\"value\")\n",
    "\n",
    "# Display our widgets\n",
    "widgets.VBox([beach_container, widgets.VBox(heatmaps)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-11-27T03:27:35.146616Z",
     "start_time": "2018-11-27T03:27:35.112525Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[-0.0, -0.0, -0.0, 0.0],\n",
       " [-0.0, -0.0, 0.0, -0.0],\n",
       " [-23.0, 14.0, -0.0, -0.0],\n",
       " [50.0, -16.0, -6.0, -0.0]]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_site_ids = df_sites[df_sites.beach.isin(beach_select.value)].index.tolist()\n",
    "\n",
    "df_ob = impacts['observed']\n",
    "df_fo = impacts['forecasted']['foreshore_slope_sto06']\n",
    "\n",
    "observed_regimes = df_ob[df_ob.index.isin(selected_site_ids)].storm_regime.dropna().rename(\"observed_regime\")\n",
    "forecasted_regimes = df_fo[df_fo.index.isin(selected_site_ids)].storm_regime.dropna().rename(\"forecasted_regime\")\n",
    "\n",
    "df_compare = pd.concat([observed_regimes, forecasted_regimes], axis='columns', names=['a','b'], sort=True)\n",
    "df_compare.dropna(axis='index',inplace=True)\n",
    "\n",
    "z = confusion_matrix(df_compare.observed_regime.tolist(), df_compare.forecasted_regime.tolist(), labels = ['swash','collision','overwash','inundation'])\n",
    "z = np.flip(z,axis=0)\n",
    "z_list = list(reversed(z.tolist()))\n",
    "\n",
    "# Make incorrect values negative, so they get assigned a different color.\n",
    "# Better for visualization\n",
    "z_neg_incorrect = np.flip(np.identity(4),axis=0)\n",
    "z_neg_incorrect[z_neg_incorrect==0]= -1\n",
    "z_neg_incorrect = (z * z_neg_incorrect).tolist()\n",
    "z_neg_incorrect"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "47px",
    "width": "262px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}