Merge branch 'feature/twl-exceedence-time' into develop

Makefile

@@ -118,6 +118,7 @@ impacts: ./data/interim/impacts_forecasted_foreshore_slope_sto06.csv ./data/inte
 	--profile-features-csv "./data/interim/profile_features.csv" \
 	--runup-function "sto06" \
 	--slope "foreshore" \
+	--profile-type "prestorm" \
 	--output-file "./data/interim/twl_foreshore_slope_sto06.csv"
 
 ./data/interim/twl_mean_slope_sto06.csv: ./data/interim/waves.csv ./data/interim/tides.csv ./data/interim/profiles.csv ./data/interim/sites.csv ./data/interim/profile_features.csv
@@ -128,6 +129,7 @@ impacts: ./data/interim/impacts_forecasted_foreshore_slope_sto06.csv ./data/inte
 	--profile-features-csv "./data/interim/profile_features.csv" \
 	--runup-function "sto06" \
 	--slope "mean" \
+	--profile-type "prestorm" \
 	--output-file "./data/interim/twl_mean_slope_sto06.csv"
 
 ./data/interim/twl_poststorm_mean_slope_sto06.csv: ./data/interim/waves.csv ./data/interim/tides.csv ./data/interim/profiles.csv ./data/interim/sites.csv ./data/interim/profile_features.csv

src/analysis/forecast_twl.py

@@ -23,7 +23,7 @@ def forecast_twl(
     runup_function,
     n_processes=MULTIPROCESS_THREADS,
     slope="foreshore",
-    profile_type='prestorm'
+    profile_type="prestorm",
 ):
     # Use df_waves as a base
     df_twl = df_waves.copy()
@@ -46,19 +46,20 @@ def forecast_twl(
         df_twl["beta"] = pd.concat(results)
 
     elif slope == "mean":
-        df_temp = df_twl.join(df_profile_features.query("profile_type=='{}'".format(profile_type)).reset_index(
-            level='profile_type')
-                              , how="inner")
+        df_temp = df_twl.join(
+            df_profile_features.query("profile_type=='{}'".format(profile_type)).reset_index(level="profile_type"),
+            how="inner",
+        )
         df_temp["mhw"] = 0.5
         with Pool(processes=n_processes) as pool:
             results = pool.starmap(
-                mean_slope_for_site_id, [(site_id, df_temp, df_profiles, "dune_toe_z", "dune_toe_x", "mhw") for
-                                         site_id in site_ids]
+                mean_slope_for_site_id,
+                [(site_id, df_temp, df_profiles, "dune_toe_z", "dune_toe_x", "mhw") for site_id in site_ids],
             )
         df_twl["beta"] = pd.concat(results)
 
     # Estimate runup
-    R2, setup, S_total, S_inc, S_ig = runup_function(df_twl, Hs0_col="Hs0", Tp_col="Tp", beta_col="beta")
+    R2, setup, S_total, S_inc, S_ig = runup_function(Hs0=df_twl['Hs0'].tolist(), Tp=df_twl["Tp"].tolist(), beta=df_twl["beta"].tolist())
 
     df_twl["R2"] = R2
     df_twl["setup"] = setup
@@ -69,13 +70,14 @@ def forecast_twl(
     df_twl["R_low"] = df_twl["tide"] + 1.1 * df_twl["setup"] - 1.1 / 2 * df_twl["S_total"]
 
     # Drop unneeded columns
-    df_twl.drop(columns=["E", "Exs", "P", "Pxs", "dir"], inplace=True, errors="ignore")
+    # df_twl.drop(columns=["E", "Exs", "P", "Pxs", "dir"], inplace=True, errors="ignore")
 
     return df_twl
 
 
-def mean_slope_for_site_id(site_id, df_twl, df_profiles, top_elevation_col, top_x_col, btm_elevation_col,
-                           profile_type='prestorm'):
+def mean_slope_for_site_id(
+    site_id, df_twl, df_profiles, top_elevation_col, top_x_col, btm_elevation_col, profile_type="prestorm"
+):
     """
     Calculates the foreshore slope values a given site_id. Returns a series (with same indicies as df_twl) of
     foreshore slopes. This function is used to parallelize getting foreshore slopes as it is computationally
@@ -100,7 +102,7 @@ def mean_slope_for_site_id(site_id, df_twl, df_profiles, top_elevation_col, top_
             top_elevation=row[top_elevation_col],
             btm_elevation=row[btm_elevation_col],
             method="end_points",
-            top_x= row[top_x_col]
+            top_x=row[top_x_col],
         ),
         axis=1,
     )
@@ -130,7 +132,7 @@ def foreshore_slope_for_site_id(site_id, df_twl, df_profiles):
             profile_x=profile_x,
             profile_z=profile_z,
             tide=row.tide,
-            runup_function=runup_models.sto06_individual,
+            runup_function=runup_models.sto06,
             Hs0=row.Hs0,
             Tp=row.Tp,
         ),
@@ -216,16 +218,14 @@ def slope_from_profile(profile_x, profile_z, top_elevation, btm_elevation, metho
 
     end_points = {"top": {"z": top_elevation}, "btm": {"z": btm_elevation}}
 
-
-
     for end_type in end_points.keys():
 
         # Add x coordinates if they are specified
-        if top_x and end_type == 'top':
-            end_points['top']['x'] = top_x
+        if top_x and end_type == "top":
+            end_points["top"]["x"] = top_x
             continue
-        if btm_x and end_type == 'top':
-            end_points['btm']['x'] = btm_x
+        if btm_x and end_type == "top":
+            end_points["btm"]["x"] = btm_x
             continue
 
         elevation = end_points[end_type]["z"]
@@ -306,14 +306,15 @@ def crossings(profile_x, profile_z, constant_z):
 @click.option("--slope", required=True, help="", type=click.Choice(["foreshore", "mean"]))
 @click.option("--profile-type", required=True, help="", type=click.Choice(["prestorm", "poststorm"]))
 @click.option("--output-file", required=True, help="")
-def create_twl_forecast(waves_csv, tides_csv, profiles_csv, profile_features_csv, runup_function, slope, 
-                        profile_type,output_file):
+def create_twl_forecast(
+    waves_csv, tides_csv, profiles_csv, profile_features_csv, runup_function, slope, profile_type, output_file
+):
     logger.info("Creating forecast of total water levels")
     logger.info("Importing data")
     df_waves = pd.read_csv(waves_csv, index_col=[0, 1])
     df_tides = pd.read_csv(tides_csv, index_col=[0, 1])
     df_profiles = pd.read_csv(profiles_csv, index_col=[0, 1, 2])
-    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0,1])
+    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0, 1])
 
     logger.info("Forecasting TWL")
     df_twl = forecast_twl(
@@ -323,7 +324,7 @@ def create_twl_forecast(waves_csv, tides_csv, profiles_csv, profile_features_csv
         df_profile_features,
         runup_function=getattr(runup_models, runup_function),
         slope=slope,
-        profile_type=profile_type
+        profile_type=profile_type,
     )
 
     df_twl.to_csv(output_file)

src/analysis/forecasted_storm_impacts.py

@@ -20,7 +20,7 @@ def forecasted_impacts(df_profile_features, df_forecasted_twl):
     """
     logger.info("Getting forecasted storm impacts")
 
-    df_forecasted_impacts = pd.DataFrame(index=df_profile_features.index.get_level_values('site_id').unique())
+    df_forecasted_impacts = pd.DataFrame(index=df_profile_features.index.get_level_values("site_id").unique())
 
     # For each site, find the maximum R_high value and the corresponding R_low value.
     idx = df_forecasted_twl.groupby(level=["site_id"])["R_high"].idxmax().dropna()
@@ -29,7 +29,12 @@ def forecasted_impacts(df_profile_features, df_forecasted_twl):
 
     # Join with df_profile features to find dune toe and crest elevations
     df_forecasted_impacts = df_forecasted_impacts.merge(
-        df_profile_features.query("profile_type=='prestorm'")[["dune_toe_z", "dune_crest_z"]], how="left", left_index=True, right_index=True
+        df_profile_features.query("profile_type=='prestorm'").reset_index("profile_type")[
+            ["dune_toe_z", "dune_crest_z"]
+        ],
+        how="left",
+        left_on=["site_id"],
+        right_on=["site_id"],
     )
 
     # Compare R_high and R_low wirth dune crest and toe elevations
@@ -66,6 +71,36 @@ def storm_regime(df_forecasted_impacts):
     return df_forecasted_impacts
 
 
+def twl_exceedence_time(df_profile_features, df_forecasted_twl, z_twl_col="R_high", z_exceedence_col="dune_toe_z"):
+    """
+    Returns a dataframe of number of hours the twl exceeded a certain z elevation.
+    May need to use this https://stackoverflow.com/a/53656968 if datetimes are not consistent.
+    :param df_profile_features:
+    :param df_forecasted_twl:
+    :param z_twl_col:
+    :param z_exceedence_col:
+    :return:
+    """
+    logger.info("Getting twl exceedence time")
+
+    # Get a dataframe of prestorm dune toes organised by site_id
+    df_dune_toes = (
+        df_profile_features.query('profile_type=="prestorm"').reset_index("profile_type")[z_exceedence_col].to_frame()
+    )
+
+    # Merge dune toes into site_id
+    df_merged = df_forecasted_twl.merge(df_dune_toes, left_on=["site_id"], right_on=["site_id"])
+
+    # Return the sum of hours that twl exceeded the level
+    return (
+        (df_merged[z_twl_col] >= df_merged[z_exceedence_col])
+        .groupby("site_id")
+        .sum()
+        .rename("twl_{}_exceedance_hrs".format(z_exceedence_col))
+        .to_frame()
+    )
+
+
 @click.command()
 @click.option("--profile-features-csv", required=True, help="")
 @click.option("--forecasted-twl-csv", required=True, help="")
@@ -73,10 +108,15 @@ def storm_regime(df_forecasted_impacts):
 def create_forecasted_impacts(profile_features_csv, forecasted_twl_csv, output_file):
     logger.info("Creating observed wave impacts")
     logger.info("Importing existing data")
-    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0,1])
+    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0, 1])
     df_forecasted_twl = pd.read_csv(forecasted_twl_csv, index_col=[0, 1])
 
     df_forecasted_impacts = forecasted_impacts(df_profile_features, df_forecasted_twl)
+
+    df_forecasted_impacts = df_forecasted_impacts.merge(
+        twl_exceedence_time(df_profile_features, df_forecasted_twl), left_on=["site_id"], right_on=["site_id"]
+    )
+
     df_forecasted_impacts.to_csv(output_file)
     logger.info("Saved to %s", output_file)
     logger.info("Done!")

src/analysis/observed_storm_impacts.py

@@ -30,13 +30,13 @@ def volume_change(df_profiles, df_profile_features, zone):
     """
     logger.info("Calculating change in beach volume in {} zone".format(zone))
 
-    df_vol_changes = pd.DataFrame(index=df_profile_features.index.get_level_values('site_id').unique())
+    df_vol_changes = pd.DataFrame(index=df_profile_features.index.get_level_values("site_id").unique())
     df_profiles = df_profiles.sort_index()
     sites = df_profiles.groupby(level=["site_id"])
 
     for site_id, df_site in sites:
         logger.debug("Calculating change in beach volume at {} in {} zone".format(site_id, zone))
-        query ="site_id=='{}'&profile_type=='prestorm'".format(site_id)
+        query = "site_id=='{}'&profile_type=='prestorm'".format(site_id)
         prestorm_dune_toe_x = df_profile_features.query(query).dune_toe_x.tolist()
         prestorm_dune_crest_x = df_profile_features.query(query).dune_crest_x.tolist()
 
@@ -71,11 +71,11 @@ def volume_change(df_profiles, df_profile_features, zone):
 
         # Where we want to measure pre and post storm volume is dependant on the zone selected
         if zone == "swash":
-            x_min = max(prestorm_dune_toe_x,x_first_obs)
+            x_min = max(prestorm_dune_toe_x, x_first_obs)
             x_max = x_last_obs
         elif zone == "dune_face":
             x_min = max(prestorm_dune_crest_x, x_first_obs)
-            x_max = min(prestorm_dune_toe_x,x_last_obs)
+            x_max = min(prestorm_dune_toe_x, x_last_obs)
         else:
             logger.warning("Zone argument not properly specified. Please check")
             x_min = None
@@ -98,16 +98,11 @@ def volume_change(df_profiles, df_profile_features, zone):
 
         # Volume change needs to be calculated including a tolerance for LIDAR accuracy. If difference between
         # profiles is less than 20 cm, consider them as zero difference.
-        prestorm_z = df_zone.query("profile_type=='prestorm'").reset_index('profile_type').z
-        poststorm_z = df_zone.query("profile_type=='poststorm'").reset_index('profile_type').z
+        prestorm_z = df_zone.query("profile_type=='prestorm'").reset_index("profile_type").z
+        poststorm_z = df_zone.query("profile_type=='poststorm'").reset_index("profile_type").z
         diff_z = prestorm_z - poststorm_z
         diff_z[abs(diff_z) < 0.2] = 0
-        diff_vol = beach_volume(
-            x=diff_z.index.get_level_values("x"),
-            z=diff_z,
-            x_min=x_min,
-            x_max=x_max,
-        )
+        diff_vol = beach_volume(x=diff_z.index.get_level_values("x"), z=diff_z, x_min=x_min, x_max=x_max)
 
         df_vol_changes.loc[site_id, "prestorm_{}_vol".format(zone)] = prestorm_vol
         df_vol_changes.loc[site_id, "poststorm_{}_vol".format(zone)] = poststorm_vol
@@ -146,14 +141,13 @@ def storm_regime(df_observed_impacts):
     logger.info("Getting observed storm regimes")
 
     swash = (df_observed_impacts.dune_face_pct_change <= 2) & (df_observed_impacts.dune_face_vol_change <= 3)
-    collision = (df_observed_impacts.dune_face_pct_change >= 2) |(df_observed_impacts.dune_face_vol_change > 3)
+    collision = (df_observed_impacts.dune_face_pct_change >= 2) | (df_observed_impacts.dune_face_vol_change > 3)
 
     df_observed_impacts.loc[swash, "storm_regime"] = "swash"
     df_observed_impacts.loc[collision, "storm_regime"] = "collision"
     return df_observed_impacts
 
 
-
 @click.command()
 @click.option("--profiles-csv", required=True, help="")
 @click.option("--profile-features-csv", required=True, help="")
@@ -163,10 +157,10 @@ def create_observed_impacts(profiles_csv, profile_features_csv, output_file):
     logger.info("Creating observed wave impacts")
     logger.info("Importing data")
     df_profiles = pd.read_csv(profiles_csv, index_col=[0, 1, 2])
-    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0,1])
+    df_profile_features = pd.read_csv(profile_features_csv, index_col=[0, 1])
 
     logger.info("Creating new dataframe for observed impacts")
-    df_observed_impacts = pd.DataFrame(index=df_profile_features.index.get_level_values('site_id').unique())
+    df_observed_impacts = pd.DataFrame(index=df_profile_features.index.get_level_values("site_id").unique())
 
     logger.info("Getting pre/post storm volumes")
     df_swash_vol_changes = volume_change(df_profiles, df_profile_features, zone="swash")
@@ -177,7 +171,7 @@ def create_observed_impacts(profiles_csv, profile_features_csv, output_file):
     df_observed_impacts = storm_regime(df_observed_impacts)
 
     # Save dataframe to csv
-    df_observed_impacts.to_csv(output_file, float_format='%.4f')
+    df_observed_impacts.to_csv(output_file, float_format="%.4f")
 
     logger.info("Saved to %s", output_file)
     logger.info("Done!")

src/analysis/runup_models.py

@@ -2,52 +2,51 @@ import numpy as np
 import pandas as pd
 
 
-def sto06_individual(Hs0, Tp, beta):
+def sto06(Hs0, Tp, beta):
+    """
+    :param Hs0: List or float of offshore significant wave height values
+    :param Tp: List or float of peak wave period
+    :param beta: List of float of beach slope
+    :return: Float or list of R2, setup, S_total, S_inc and S_ig values
+    """
 
-    Lp = 9.8 * Tp ** 2 / 2 / np.pi
+    df = pd.DataFrame({"Hs0": Hs0, "Tp": Tp, "beta": beta}, index=[x for x in range(0, np.size(Hs0))])
 
-    S_ig = 0.06 * np.sqrt(Hs0 * Lp)
-    S_inc = 0.75 * beta * np.sqrt(Hs0 * Lp)
+    df["Lp"] = 9.8 * df['Tp'] ** 2 / 2 / np.pi
+
+    # General equation
+    df["S_ig"] = pd.to_numeric(0.06 * np.sqrt(df["Hs0"] * df["Lp"]), errors="coerce")
+    df["S_inc"] = pd.to_numeric(0.75 * df["beta"] * np.sqrt(df["Hs0"] * df["Lp"]), errors="coerce")
+    df["setup"] = pd.to_numeric(0.35 * df["beta"] * np.sqrt(df["Hs0"] * df["Lp"]), errors="coerce")
+    df["S_total"] = np.sqrt(df["S_inc"] ** 2 + df["S_ig"] ** 2)
+    df["R2"] = 1.1 * (df["setup"] + df["S_total"] / 2)
 
     # Dissipative conditions
-    if beta / (Hs0 / Lp) ** (0.5) <= 0.3:
-        setup = 0.016 * (Hs0 * Lp) ** 0.5
-        S_total = 0.046 * (Hs0 * Lp) ** 0.5
-        R2 = 0.043 * (Hs0 * Lp) ** 0.5
-    else:
-        setup = 0.35 * beta * (Hs0 * Lp) ** 0.5
-        S_total = np.sqrt(S_inc ** 2 + S_ig ** 2)
-        R2 = 1.1 * (setup + S_total / 2)
+    dissipative = df["beta"] / (df["Hs0"] / df["Lp"]) ** (0.5) <= 0.3
+
+    df.loc[dissipative, "setup"] = 0.016 * (df["Hs0"] * df["Lp"]) ** (0.5)  # eqn 16
+    df.loc[dissipative, "S_total"] = 0.046 * (df["Hs0"] * df["Lp"]) ** (0.5)  # eqn 17
+    df.loc[dissipative, "R2"] = 0.043 * (df["Hs0"] * df["Lp"]) ** (0.5)  # eqn 18
 
-    return R2, setup, S_total, S_inc, S_ig
+    return (
+        float_or_list(df["R2"].tolist()),
+        float_or_list(df["setup"].tolist()),
+        float_or_list(df["S_total"].tolist()),
+        float_or_list(df["S_inc"].tolist()),
+        float_or_list(df["S_ig"].tolist()),
+    )
 
 
-def sto06(df, Hs0_col, Tp_col, beta_col):
+def float_or_list(a):
     """
-    Vectorized version of Stockdon06 which can be used with dataframes
-    :param df:
-    :param Hs0_col:
-    :param Tp_col:
-    :param beta_col:
+    If only one value in the array, return the float, else return a list
+    :param a:
     :return:
     """
-
-    Lp = 9.8 * df[Tp_col] ** 2 / 2 / np.pi
-
-    # General equation
-    S_ig = pd.to_numeric(0.06 * np.sqrt(df[Hs0_col] * Lp), errors="coerce")
-    S_inc = pd.to_numeric(0.75 * df[beta_col] * np.sqrt(df[Hs0_col] * Lp), errors="coerce")
-    setup = pd.to_numeric(0.35 * df[beta_col] * np.sqrt(df[Hs0_col] * Lp), errors="coerce")
-    S_total = np.sqrt(S_inc ** 2 + S_ig ** 2)
-    R2 = 1.1 * (setup + S_total / 2)
-
-    # Dissipative conditions
-    dissipative = df[beta_col] / (df[Hs0_col] / Lp) ** (0.5) <= 0.3
-    setup.loc[dissipative, :] = 0.016 * (df[Hs0_col] * Lp) ** (0.5)  # eqn 16
-    S_total.loc[dissipative, :] = 0.046 * (df[Hs0_col] * Lp) ** (0.5)  # eqn 17
-    R2.loc[dissipative, :] = 0.043 * (df[Hs0_col] * Lp) ** (0.5)  # eqn 18
-
-    return R2, setup, S_total, S_inc, S_ig
+    if len(a) == 1:
+        return a[0]
+    else:
+        return list(a)
 
 
 if __name__ == "__main__":

src/cli.py

@@ -13,7 +13,9 @@ import analysis.observed_storm_impacts as observed_storm_impacts
 
 # Disable numpy warnings
 import warnings
-warnings.simplefilter(action='ignore', category=FutureWarning)
+
+warnings.simplefilter(action="ignore", category=FutureWarning)
+
 
 @click.group()
 def cli():

src/data/parse_mat.py

@@ -58,35 +58,41 @@ def parse_dune_crest_toes(df_sites, crest_mat, toe_mat):
     crest_data = loadmat(crest_mat)
     toe_data = loadmat(toe_mat)
 
-    for n, _ in enumerate(crest_data['xc1']):
-        rows.extend([{
-            'dune_crest_x': crest_data['xc1'][n],
-            'dune_crest_z': crest_data['zc1'][n],
-            'dune_toe_x': toe_data['xt1'][n],
-            'dune_toe_z': toe_data['zt1'][n],
-            'profile_type': 'prestorm',
-            'site_no': n+1
-        },{
-            'dune_crest_x': crest_data['xc2'][n],
-            'dune_crest_z': crest_data['zc2'][n],
-            'dune_toe_x': toe_data['xt2'][n],
-            'dune_toe_z': toe_data['zt2'][n],
-            'profile_type': 'poststorm',
-            'site_no': n + 1
-        }])
+    for n, _ in enumerate(crest_data["xc1"]):
+        rows.extend(
+            [
+                {
+                    "dune_crest_x": crest_data["xc1"][n],
+                    "dune_crest_z": crest_data["zc1"][n],
+                    "dune_toe_x": toe_data["xt1"][n],
+                    "dune_toe_z": toe_data["zt1"][n],
+                    "profile_type": "prestorm",
+                    "site_no": n + 1,
+                },
+                {
+                    "dune_crest_x": crest_data["xc2"][n],
+                    "dune_crest_z": crest_data["zc2"][n],
+                    "dune_toe_x": toe_data["xt2"][n],
+                    "dune_toe_z": toe_data["zt2"][n],
+                    "profile_type": "poststorm",
+                    "site_no": n + 1,
+                },
+            ]
+        )
 
     df_profile_features = pd.DataFrame(rows)
 
     # Want the site_id instead of the site_no, so merge in df_sites
     df_sites.reset_index(inplace=True)
-    df_profile_features = df_sites[['site_no','site_id']].merge(df_profile_features, how='outer', on=['site_no'])
-    df_profile_features.drop(columns=['site_no'],inplace=True)
-    df_profile_features.set_index(['site_id','profile_type'], inplace=True)
+    df_profile_features = df_sites[["site_no", "site_id"]].merge(df_profile_features, how="outer", on=["site_no"])
+    df_profile_features.drop(columns=["site_no"], inplace=True)
+    df_profile_features.set_index(["site_id", "profile_type"], inplace=True)
     df_profile_features.sort_index(inplace=True)
     df_profile_features = df_profile_features.round(3)
 
     return df_profile_features
 
+
 def combine_sites_and_orientaions(df_sites, df_orientations):
     """
     Replaces beach/lat/lon columns with the unique site_id.
@@ -193,7 +199,7 @@ def parse_profiles_and_sites(profiles_mat):
     site_counter = 0
 
     for i, site in enumerate(mat_data["site"]):
-        logger.debug('Processing site {} of {}'.format(i+1, len(mat_data['site'])))
+        logger.debug("Processing site {} of {}".format(i + 1, len(mat_data["site"])))
 
         # Give each site a unique id
         if len(site_rows) == 0 or site_rows[-1]["beach"] != site:
@@ -248,7 +254,6 @@ def parse_profiles_and_sites(profiles_mat):
                     profile_rows.append(
                         {
                             "site_id": site_id,
-
                             "lon": lon[0],
                             "lat": lat[0],
                             "profile_type": profile_type,
@@ -387,6 +392,7 @@ def create_profile_features(crest_mat, toe_mat, sites_csv, output_file):
     df_profile_features.to_csv(output_file)
     logger.info("Created %s", output_file)
 
+
 @click.command(short_help="create profiles.csv")
 @click.option("--profiles-mat", required=True, help=".mat file containing beach profiles")
 @click.option("--profiles-output-file", required=True, help="where to save profiles.csv")